User Inputs

output.var = params$output.var 

transform.abs = FALSE
log.pred = params$log.pred
norm.pred = FALSE
eda = params$eda
algo.forward.caret = params$algo.forward.caret
algo.backward.caret = params$algo.backward.caret
algo.stepwise.caret = params$algo.stepwise.caret
algo.LASSO.caret = params$algo.LASSO.caret
algo.LARS.caret = params$algo.LARS.caret
message("Parameters used for training/prediction: ")
## Parameters used for training/prediction:
str(params)
## List of 8
##  $ output.var         : chr "y3"
##  $ log.pred           : logi TRUE
##  $ eda                : logi FALSE
##  $ algo.forward.caret : logi TRUE
##  $ algo.backward.caret: logi TRUE
##  $ algo.stepwise.caret: logi TRUE
##  $ algo.LASSO.caret   : logi TRUE
##  $ algo.LARS.caret    : logi TRUE
# Setup Labels
output.var.tr = if (log.pred == TRUE)  paste0(output.var,'.log') else  output.var.tr = output.var

Loading Data

feat  = read.csv('../../Data/features_highprec.csv')
labels = read.csv('../../Data/labels.csv')
predictors = names(dplyr::select(feat,-JobName))
data.ori = inner_join(feat,labels,by='JobName')
#data.ori = inner_join(feat,select_at(labels,c('JobName',output.var)),by='JobName')

Data validation

cc  = complete.cases(data.ori)
data.notComplete = data.ori[! cc,]
data = data.ori[cc,] %>% select_at(c(predictors,output.var,'JobName'))
message('Original cases: ',nrow(data.ori))
## Original cases: 10000
message('Non-Complete cases: ',nrow(data.notComplete))
## Non-Complete cases: 3020
message('Complete cases: ',nrow(data))
## Complete cases: 6980
summary(dplyr::select_at(data,c('JobName',output.var)))
##       JobName           y3        
##  Job_00001:   1   Min.   : 95.91  
##  Job_00002:   1   1st Qu.:118.29  
##  Job_00003:   1   Median :124.03  
##  Job_00004:   1   Mean   :125.40  
##  Job_00007:   1   3rd Qu.:131.06  
##  Job_00008:   1   Max.   :193.73  
##  (Other)  :6974

Output Variable

The Output Variable y3 shows right skewness, so will proceed with a log transformation

Histogram

df=gather(select_at(data,output.var))
ggplot(df, aes(x=value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() 

  #stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  

QQPlot

ggplot(gather(select_at(data,output.var)), aes(sample=value)) + 
  stat_qq() + 
  facet_wrap(~key, scales = 'free',ncol=4)

Trasformation of Output Variable from y3 to y3.log

if(log.pred==TRUE) data[[output.var.tr]] = log(data[[output.var]],10) else
  data[[output.var.tr]] = data[[output.var]]
df=gather(select_at(data,c(output.var,output.var.tr)))
ggplot(df, aes(value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() + 
  # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
  facet_wrap(~key, scales = 'free',ncol=2)

ggplot(gather(select_at(data,c(output.var,output.var.tr))), aes(sample=value)) + 
  stat_qq() + 
  facet_wrap(~key, scales = 'free',ncol=4)

Best Normalizator y3

Normalization of y3 using bestNormalize package. (suggested orderNorm) This is cool, but I think is too far for the objective of the project

t=bestNormalize::bestNormalize(data[[output.var]])
t
## Best Normalizing transformation with 6980 Observations
##  Estimated Normality Statistics (Pearson P / df, lower => more normal):
##  - No transform: 2.9627 
##  - Box-Cox: 1.426 
##  - Log_b(x+a): 1.9884 
##  - sqrt(x+a): 2.4513 
##  - exp(x): 749.4167 
##  - arcsinh(x): 1.9884 
##  - Yeo-Johnson: 1.1169 
##  - orderNorm: 1.1737 
## Estimation method: Out-of-sample via CV with 10 folds and 5 repeats
##  
## Based off these, bestNormalize chose:
## Standardized Yeo-Johnson Transformation with 6980 nonmissing obs.:
##  Estimated statistics:
##  - lambda = -1.998639 
##  - mean (before standardization) = 0.5003083 
##  - sd (before standardization) = 5.108542e-06
qqnorm(data[[output.var]])

qqnorm(predict(t))

orderNorm() is a rank-based procedure by which the values of a vector are mapped to their percentile, which is then mapped to the same percentile of the normal distribution. Without the presence of ties, this essentially guarantees that the transformation leads to a uniform distribution

Predictors

All predictors show a Fat-Tail situation, where the two tails are very tall, and a low distribution around the mean. The orderNorm transformation can help (see [Best Normalizator] section)

Interesting Predictors

Histograms

if (eda == TRUE){
  cols = c('x11','x18','stat98','x7','stat110')
  df=gather(select_at(data,cols))
  ggplot(df, aes(value)) + 
    geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
    geom_density() + 
    # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
    facet_wrap(~key, scales = 'free',ncol=3)
  
  # ggplot(gather(select_at(data,cols)), aes(sample=value)) + 
  #   stat_qq()+
  #   facet_wrap(~key, scales = 'free',ncol=2)
  
  lapply(select_at(data,cols),summary)
}

Scatter plot vs. output variable **y3.log

if (eda == TRUE){
  d = gather(dplyr::select_at(data,c(cols,output.var.tr)),key=target,value=value,-!!output.var.tr)
  ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
    geom_point(color='light green',alpha=0.5) + 
    geom_smooth() + 
    facet_wrap(~target, scales = 'free',ncol=3)
}

All Predictors

Histograms

All indicators have a strong indication of Fat-Tails

if (eda == TRUE){
  df=gather(select_at(data,predictors))
  ggplot(df, aes(value)) + 
    geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
    geom_density() + 
    # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
    facet_wrap(~key, scales = 'free',ncol=4)
}

Correlations

With Output Variable

if (eda == TRUE){
  #chart.Correlation(select(data,-JobName),  pch=21)
  t=as.data.frame(round(cor(dplyr::select(data,-one_of(output.var.tr,'JobName'))
                            ,select_at(data,output.var.tr)),4))  %>%
    rownames_to_column(var='variable') %>% filter(variable != !!output.var) %>% arrange(-y3.log)
  #DT::datatable(t)
  message("Top Positive")
  kable(head(arrange(t,desc(y3.log)),20))
  message("Top Negative")
  kable(head(arrange(t,y3.log),20))
}

Between All Variables

if (eda == TRUE){
  #chart.Correlation(select(data,-JobName),  pch=21)
  t=as.data.frame(round(cor(dplyr::select(data,-one_of('JobName'))),4))
  #DT::datatable(t,options=list(scrollX=T))
  message("Showing only 10 variables")
  kable(t[1:10,1:10])
}

Scatter Plots with Output Variable

Scatter plots with all predictors and the output variable (y3.log)

if (eda == TRUE){
  d = gather(dplyr::select_at(data,c(predictors,output.var.tr)),key=target,value=value,-!!output.var.tr)
  ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
    geom_point(color='light blue',alpha=0.5) + 
    geom_smooth() + 
    facet_wrap(~target, scales = 'free',ncol=4)
}

Multicollinearity - VIF

No Multicollinearity among predictors

Showing Top predictor by VIF Value

if (eda == TRUE){
  vifDF = usdm::vif(select_at(data,predictors)) %>% arrange(desc(VIF))
  head(vifDF,15)
}

Feature Eng

  • Square Root transformation for x18
data.tr=data %>%
  mutate(x18.sqrt = sqrt(x18)) 
cols=c('x18','x18.sqrt')

Comparing Pre and Post Transformation Density Plots

# ggplot(gather(select_at(data.tr,cols)), aes(value)) + 
#   geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
#   geom_density() + 
#   facet_wrap(~key, scales = 'free',ncol=4)

d = gather(dplyr::select_at(data.tr,c(cols,output.var.tr)),key=target,value=value,-!!output.var.tr)
ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
  geom_point(color='light blue',alpha=0.5) + 
  geom_smooth() + 
  facet_wrap(~target, scales = 'free',ncol=4)
## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'

#removing unwanted variables
data.tr=data.tr %>%
  dplyr::select_at(names(data.tr)[! names(data.tr) %in% c('x18','y3','JobName')])

data=data.tr
label.names=output.var.tr

Modeling

PCA

# 0 for no interaction, 
# 1 for Full 2 way interaction and 
# 2 for Selective 2 way interaction
# 3 for Selective 3 way interaction
InteractionMode = 3

pca.vars  = names(data)
pca.vars = pca.vars[!pca.vars %in% label.names]

if(InteractionMode == 1){
  pca.formula =as.formula(paste0('~(',paste0(pca.vars, collapse ='+'),')^2'))
  pca.model = prcomp(formula=pca.formula,data=data[,pca.vars],center=T,scale.=T,retx = T)
  #saveRDS(pca.model,'pca.model.rds')
}
if (InteractionMode == 0){
  pca.model =  prcomp(x=data[,pca.vars],center=T,scale.=T,retx = T)
}
if (InteractionMode >= 2 & InteractionMode <= 3){
  controlled.vars = pca.vars[grep("^x",pca.vars)]
  stat.vars = pca.vars[grep("^stat",pca.vars)]
  
  if (InteractionMode >= 2){
    interaction.form = paste0('~(',paste0(controlled.vars, collapse ='+'),')^2')
  }
  if (InteractionMode >= 3){
    interaction.form = paste0('~(',paste0(controlled.vars, collapse ='+'),')^3')
  }
  no.interact.form = paste0(stat.vars, collapse ='+')
  
  pca.formula = as.formula(paste(interaction.form, no.interact.form, sep = "+"))
  pca.model = prcomp(formula=pca.formula,data=data[,pca.vars],center=T,scale.=T,retx = T)
}
targetCumVar = .9

pca.model$var = pca.model$sdev ^ 2 #eigenvalues
pca.model$pvar = pca.model$var / sum(pca.model$var)
pca.model$cumpvar = cumsum(pca.model$pvar )
pca.model$pcaSel = pca.model$cumpvar<=targetCumVar
pca.model$pcaSelCount = sum(pca.model$pcaSel)
pca.model$pcaSelTotVar = sum(pca.model$pvar[pca.model$pcaSel])
message(pca.model$pcaSelCount, " PCAs justify ",percent(targetCumVar)," of the total Variance. (",percent(pca.model$pcaSelTotVar),")")
## 156 PCAs justify 90.0% of the total Variance. (90.0%)
plot(pca.model$var,xlab="Principal component", ylab="Proportion of variance explained",   type='b')

plot(cumsum(pca.model$pvar ),xlab="Principal component", ylab="Cumulative Proportion of variance explained", ylim=c(0,1), type='b')

screeplot(pca.model,npcs = pca.model$pcaSelCount)

screeplot(pca.model,npcs = pca.model$pcaSelCount,type='lines')

#summary(pca.model)
#pca.model$rotation
#creating dataset
data.pca = dplyr::select(data,!!label.names) %>% 
  dplyr::bind_cols(dplyr::select(as.data.frame(pca.model$x)
                                 ,!!colnames(pca.model$rotation)[pca.model$pcaSel])
  )

Train Test Split

data.pca = data.pca[sample(nrow(data.pca)),] # randomly shuffle data
split = sample.split(data.pca[,label.names], SplitRatio = 0.8)

data.train = subset(data.pca, split == TRUE)
data.test = subset(data.pca, split == FALSE)

Common Functions

plot.diagnostics <-  function(model, train) {
  plot(model)
  
  residuals = resid(model) # Plotted above in plot(lm.out)
  r.standard = rstandard(model)
  r.student = rstudent(model)
  
  df = data.frame(x=predict(model,train),y=r.student)
  p=ggplot(data=df,aes(x=x,y=y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_hline(yintercept = 0,size=1)+
    ylab("Student Residuals") +
    xlab("Predicted Values")+
    ggtitle("Student Residual Plot")
  plot(p)
  
  df = data.frame(x=predict(model,train),y=r.standard)
  p=ggplot(data=df,aes(x=x,y=y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_hline(yintercept = c(-2,0,2),size=1)+
    ylab("Student Residuals") +
    xlab("Predicted Values")+
    ggtitle("Student Residual Plot")
  plot(p)
  # Histogram
  df=data.frame(r.student)
  p=ggplot(data=df,aes(r.student)) +
    geom_histogram(aes(y=..density..),bins = 50,fill='blue',alpha=0.6) + 
    stat_function(fun = dnorm, n = 100, args = list(mean = 0, sd = 1)) +
    ylab("Density")+
    xlab("Studentized Residuals")+
    ggtitle("Distribution of Studentized Residuals")
  plot(p)
  # http://www.stat.columbia.edu/~martin/W2024/R7.pdf
  # Influential plots
  inf.meas = influence.measures(model)
  # print (summary(inf.meas)) # too much data
  
  # Leverage plot
  lev = hat(model.matrix(model))
  df=tibble::rownames_to_column(as.data.frame(lev),'id')
  p=ggplot(data=df,aes(x=as.numeric(id),y=lev)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    ylab('Leverage - check') + 
    xlab('Index')
  plot(p)
  # Cook's Distance
  cd = cooks.distance(model)
  df=tibble::rownames_to_column(as.data.frame(cd),'id')
  p=ggplot(data=df,aes(x=as.numeric(id),y=cd)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_text(data=filter(df,cd>15/nrow(train)),aes(label=id),check_overlap=T,size=3,vjust=-.5)+
    ylab('Cooks distances') + 
    geom_hline(yintercept = c(4/nrow(train),0),size=1)+
    xlab('Index')
  plot(p)
  print (paste("Number of data points that have Cook's D > 4/n: ", length(cd[cd > 4/nrow(train)]), sep = "")) 
  print (paste("Number of data points that have Cook's D > 1: ", length(cd[cd > 1]), sep = "")) 
  return(cd)
}

# function to set up random seeds
# Based on http://jaehyeon-kim.github.io/2015/05/Setup-Random-Seeds-on-Caret-Package.html 
setCaretSeeds <- function(method = "cv", numbers = 1, repeats = 1, tunes = NULL, seed = 1701) {
  #B is the number of resamples and integer vector of M (numbers + tune length if any)
  B <- if (method == "cv") numbers
  else if(method == "repeatedcv") numbers * repeats
  else NULL
  if(is.null(length)) {
    seeds <- NULL
  } else {
    set.seed(seed = seed)
    seeds <- vector(mode = "list", length = B)
    seeds <- lapply(seeds, function(x) sample.int(n = 1000000
                                                  , size = numbers + ifelse(is.null(tunes), 0, tunes)))
    seeds[[length(seeds) + 1]] <- sample.int(n = 1000000, size = 1)
  }
  # return seeds
  seeds
}



train.caret.glmselect = function(formula, data, method
                                 ,subopt = NULL, feature.names
                                 , train.control = NULL, tune.grid = NULL, pre.proc = NULL){
  
  if(is.null(train.control)){
    train.control <- trainControl(method = "cv"
                              ,number = 10
                              ,seeds = setCaretSeeds(method = "cv"
                                                     , numbers = 10
                                                     , seed = 1701)
                              ,search = "grid"
                              ,verboseIter = TRUE
                              ,allowParallel = TRUE
                              )
  }
  
  if(is.null(tune.grid)){
    if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
      tune.grid = data.frame(nvmax = 1:length(feature.names))
    }
    if (method == 'glmnet' && subopt == 'LASSO'){
      # Will only show 1 Lambda value during training, but that is OK
      # https://stackoverflow.com/questions/47526544/why-need-to-tune-lambda-with-carettrain-method-glmnet-and-cv-glmnet
      # Another option for LASSO is this: https://github.com/topepo/caret/blob/master/RegressionTests/Code/lasso.R
      lambda = 10^seq(-2,0, length =100)
      alpha = c(1)
      tune.grid = expand.grid(alpha = alpha,lambda = lambda)
    }
    if (method == 'lars'){
      # https://github.com/topepo/caret/blob/master/RegressionTests/Code/lars.R
      fraction = seq(0, 1, length = 100)
      tune.grid = expand.grid(fraction = fraction)
      pre.proc = c("center", "scale") 
    }
  }
  
  # http://sshaikh.org/2015/05/06/parallelize-machine-learning-in-r-with-multi-core-cpus/
  # #cl <- makeCluster(ceiling(detectCores()*0.5)) # use 75% of cores only, leave rest for other tasks
  cl <- makeCluster(detectCores()*0.75) # use 75% of cores only, leave rest for other tasks
  registerDoParallel(cl)

  set.seed(1) 
  # note that the seed has to actually be set just before this function is called
  # settign is above just not ensure reproducibility for some reason
  model.caret <- caret::train(formula
                              , data = data
                              , method = method
                              , tuneGrid = tune.grid
                              , trControl = train.control
                              , preProc = pre.proc
                              )
  
  stopCluster(cl)
  registerDoSEQ() # register sequential engine in case you are not using this function anymore
  
  if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
    print("All models results")
    print(model.caret$results) # all model results
    print("Best Model")
    print(model.caret$bestTune) # best model
    model = model.caret$finalModel

    # Metrics Plot 
    dataPlot = model.caret$results %>%
      gather(key='metric',value='value',-nvmax) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=nvmax,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot
    # leap function does not support studentized residuals
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)
   
    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') + 
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    id = rownames(model.caret$bestTune)    
    # Provides the coefficients of the best model
    # regsubsets doens return a full model (see documentation of regsubset), so we need to recalcualte themodel
    # https://stackoverflow.com/questions/13063762/how-to-obtain-a-lm-object-from-regsubsets
    print("Coefficients of final model:")
    coefs <- coef(model, id=id)
    #calculate the model to the the coef intervals
    nams <- names(coefs)
    nams <- nams[!nams %in% "(Intercept)"]
    response <-  as.character(formula[[2]])
    form <- as.formula(paste(response, paste(nams, collapse = " + "), sep = " ~ "))
    mod <- lm(form, data = data)
    #coefs
    #coef(mod)
    print(car::Confint(mod))
    return(list(model = model,id = id, residPlot = residPlot, residHistogram=residHistogram
                ,modelLM=mod))
  }
  if (method == 'glmnet' && subopt == 'LASSO'){
    print(model.caret)
    print(plot(model.caret))
    print(model.caret$bestTune)
    
    print(model.caret$results)
    model=model.caret$finalModel
    # Metrics Plot 
    dataPlot = model.caret$results %>%
      gather(key='metric',value='value',-lambda) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=lambda,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot 
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)

    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') +
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    
    print("Coefficients") 
    #no interval for glmnet: https://stackoverflow.com/questions/39750965/confidence-intervals-for-ridge-regression
    t=coef(model,s=model.caret$bestTune$lambda)
    model.coef = t[which(t[,1]!=0),]
    print(as.data.frame(model.coef))
    id = NULL # not really needed but added for consistency
    return(list(model = model.caret,id = id, residPlot = residPlot, metricsPlot=metricsPlot ))
  }
  if (method == 'lars'){
    print(model.caret)
    print(plot(model.caret))
    print(model.caret$bestTune)
    
    # Metrics Plot
    dataPlot = model.caret$results %>%
        gather(key='metric',value='value',-fraction) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=fraction,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)

    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') + 
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    
    print("Coefficients") 
    t=coef(model.caret$finalModel,s=model.caret$bestTune$fraction,mode='fraction')
    model.coef = t[which(t!=0)]
    print(model.coef)
    id = NULL # not really needed but added for consistency
    return(list(model = model.caret,id = id, residPlot = residPlot, residHistogram=residHistogram))
  }
}

# https://stackoverflow.com/questions/48265743/linear-model-subset-selection-goodness-of-fit-with-k-fold-cross-validation
# changed slightly since call[[2]] was just returning "formula" without actually returnign the value in formula
predict.regsubsets <- function(object, newdata, id, formula, ...) {
    #form <- as.formula(object$call[[2]])
    mat <- model.matrix(formula, newdata) # adds intercept and expands any interaction terms
    coefi <- coef(object, id = id)
    xvars <- names(coefi)
    return(mat[,xvars]%*%coefi)
}
  
test.model = function(model, test, level=0.95
                      ,draw.limits = FALSE, good = 0.1, ok = 0.15
                      ,method = NULL, subopt = NULL
                      ,id = NULL, formula, feature.names, label.names
                      ,transformation = NULL){
  ## if using caret for glm select equivalent functionality, 
  ## need to pass formula (full is ok as it will select subset of variables from there)
  if (is.null(method)){
    pred = predict(model, newdata=test, interval="confidence", level = level) 
  }
  
  if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
    pred = predict.regsubsets(model, newdata = test, id = id, formula = formula)
  }
  
  if (method == 'glmnet' && subopt == 'LASSO'){
    xtest = as.matrix(test[,feature.names]) 
    pred=as.data.frame(predict(model, xtest))
  }
  
  if (method == 'lars'){
    pred=as.data.frame(predict(model, newdata = test))
  }
    
  # Summary of predicted values
  print ("Summary of predicted values: ")
  print(summary(pred[,1]))

  test.mse = mean((test[,label.names]-pred[,1])^2)
  print (paste(method, subopt, "Test MSE:", test.mse, sep=" "))
  
  if(log.pred == TRUE || norm.pred == TRUE){
    # plot transformewd comparison first
    df=data.frame(x=test[,label.names],y=pred[,1])
    ggplot(df,aes(x=x,y=y)) +
      geom_point(color='blue',alpha=0.5,shape=20,size=2) +
      geom_abline(slope=1,intercept=0,color='black',size=1) +
      #scale_y_continuous(limits=c(min(df),max(df)))+
      xlab("Actual (Transformed)")+
      ylab("Predicted (Transformed)")
  }
    
  if (log.pred == FALSE && norm.pred == FALSE){
    x = test[,label.names]
    y = pred[,1]
  }
  if (log.pred == TRUE){
    x = 10^test[,label.names]
    y = 10^pred[,1]  
  }
  if (norm.pred == TRUE){
    x = predict(transformation, test[,label.names], inverse = TRUE)
    y = predict(transformation, pred[,1], inverse = TRUE)
  }

  df=data.frame(x,y)
  ggplot(df,aes(x,y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_abline(slope=c(1+good,1-good,1+ok,1-ok)
                ,intercept=rep(0,4),color=c('dark green','dark green','dark red','dark red'),size=1,alpha=0.8) +
    #scale_y_continuous(limits=c(min(df),max(df)))+
    xlab("Actual")+
    ylab("Predicted") 
    
 
}

Setup Formulae

n <- names(data.train)
 formula <- as.formula(paste(paste(n[n %in% label.names], collapse = " + ")
                             ," ~", paste(n[!n %in% label.names], collapse = " + "))) 

grand.mean.formula = as.formula(paste(paste(n[n %in% label.names], collapse = " + ")," ~ 1"))

print(formula)
## y3.log ~ PC1 + PC2 + PC3 + PC4 + PC5 + PC6 + PC7 + PC8 + PC9 + 
##     PC10 + PC11 + PC12 + PC13 + PC14 + PC15 + PC16 + PC17 + PC18 + 
##     PC19 + PC20 + PC21 + PC22 + PC23 + PC24 + PC25 + PC26 + PC27 + 
##     PC28 + PC29 + PC30 + PC31 + PC32 + PC33 + PC34 + PC35 + PC36 + 
##     PC37 + PC38 + PC39 + PC40 + PC41 + PC42 + PC43 + PC44 + PC45 + 
##     PC46 + PC47 + PC48 + PC49 + PC50 + PC51 + PC52 + PC53 + PC54 + 
##     PC55 + PC56 + PC57 + PC58 + PC59 + PC60 + PC61 + PC62 + PC63 + 
##     PC64 + PC65 + PC66 + PC67 + PC68 + PC69 + PC70 + PC71 + PC72 + 
##     PC73 + PC74 + PC75 + PC76 + PC77 + PC78 + PC79 + PC80 + PC81 + 
##     PC82 + PC83 + PC84 + PC85 + PC86 + PC87 + PC88 + PC89 + PC90 + 
##     PC91 + PC92 + PC93 + PC94 + PC95 + PC96 + PC97 + PC98 + PC99 + 
##     PC100 + PC101 + PC102 + PC103 + PC104 + PC105 + PC106 + PC107 + 
##     PC108 + PC109 + PC110 + PC111 + PC112 + PC113 + PC114 + PC115 + 
##     PC116 + PC117 + PC118 + PC119 + PC120 + PC121 + PC122 + PC123 + 
##     PC124 + PC125 + PC126 + PC127 + PC128 + PC129 + PC130 + PC131 + 
##     PC132 + PC133 + PC134 + PC135 + PC136 + PC137 + PC138 + PC139 + 
##     PC140 + PC141 + PC142 + PC143 + PC144 + PC145 + PC146 + PC147 + 
##     PC148 + PC149 + PC150 + PC151 + PC152 + PC153 + PC154 + PC155 + 
##     PC156
print(grand.mean.formula)
## y3.log ~ 1
# Update feature.names because we may have transformed some features
feature.names = n[!n %in% label.names]

Full Model

model.full = lm(formula , data.train)
summary(model.full)
## 
## Call:
## lm(formula = formula, data = data.train)
## 
## Residuals:
##      Min       1Q   Median       3Q      Max 
## -0.09425 -0.02264 -0.00503  0.01710  0.18637 
## 
## Coefficients:
##               Estimate Std. Error  t value Pr(>|t|)    
## (Intercept)  2.097e+00  4.330e-04 4842.834  < 2e-16 ***
## PC1          2.851e-04  2.901e-05    9.828  < 2e-16 ***
## PC2         -3.788e-04  3.423e-05  -11.065  < 2e-16 ***
## PC3         -8.261e-05  4.099e-05   -2.015 0.043918 *  
## PC4         -9.391e-05  4.220e-05   -2.225 0.026117 *  
## PC5         -2.385e-04  4.231e-05   -5.637 1.82e-08 ***
## PC6          1.029e-04  4.326e-05    2.378 0.017447 *  
## PC7         -1.666e-04  4.386e-05   -3.798 0.000148 ***
## PC8         -1.319e-04  4.502e-05   -2.929 0.003417 ** 
## PC9          9.820e-05  4.669e-05    2.103 0.035477 *  
## PC10        -1.938e-05  4.817e-05   -0.402 0.687420    
## PC11         4.749e-04  4.991e-05    9.515  < 2e-16 ***
## PC12        -3.645e-04  5.021e-05   -7.259 4.44e-13 ***
## PC13         3.080e-04  5.180e-05    5.947 2.91e-09 ***
## PC14         5.437e-04  5.248e-05   10.360  < 2e-16 ***
## PC15        -1.099e-04  5.398e-05   -2.037 0.041734 *  
## PC16         2.116e-04  5.623e-05    3.764 0.000169 ***
## PC17        -8.012e-05  5.846e-05   -1.371 0.170550    
## PC18        -8.213e-05  6.058e-05   -1.356 0.175186    
## PC19        -9.287e-05  6.329e-05   -1.468 0.142292    
## PC20         3.989e-04  6.777e-05    5.885 4.21e-09 ***
## PC21        -3.618e-04  7.083e-05   -5.109 3.36e-07 ***
## PC22         1.857e-03  7.491e-05   24.791  < 2e-16 ***
## PC23        -5.132e-04  2.145e-04   -2.393 0.016762 *  
## PC24        -3.735e-05  2.345e-04   -0.159 0.873470    
## PC25         4.272e-04  2.363e-04    1.808 0.070617 .  
## PC26         1.483e-04  2.452e-04    0.605 0.545497    
## PC27        -1.627e-04  2.517e-04   -0.646 0.518153    
## PC28        -1.552e-04  2.531e-04   -0.613 0.539812    
## PC29         2.650e-04  2.626e-04    1.009 0.312931    
## PC30        -1.212e-04  2.754e-04   -0.440 0.659980    
## PC31        -2.074e-04  2.882e-04   -0.719 0.471881    
## PC32         1.819e-04  2.981e-04    0.610 0.541825    
## PC33         8.708e-06  2.992e-04    0.029 0.976780    
## PC34         3.666e-04  3.048e-04    1.203 0.229217    
## PC35         4.560e-05  3.062e-04    0.149 0.881617    
## PC36         7.130e-05  3.104e-04    0.230 0.818291    
## PC37         6.686e-04  3.161e-04    2.115 0.034471 *  
## PC38         3.433e-04  3.150e-04    1.090 0.275791    
## PC39        -7.474e-04  3.174e-04   -2.354 0.018594 *  
## PC40        -3.080e-05  3.203e-04   -0.096 0.923420    
## PC41         3.322e-04  3.225e-04    1.030 0.303124    
## PC42         2.202e-04  3.223e-04    0.683 0.494639    
## PC43         6.501e-05  3.236e-04    0.201 0.840821    
## PC44         1.178e-04  3.268e-04    0.360 0.718513    
## PC45         3.993e-04  3.275e-04    1.219 0.222865    
## PC46         2.721e-04  3.321e-04    0.819 0.412544    
## PC47        -6.007e-04  3.303e-04   -1.819 0.069032 .  
## PC48         3.744e-04  3.357e-04    1.115 0.264719    
## PC49         3.018e-04  3.344e-04    0.902 0.366869    
## PC50         6.516e-04  3.369e-04    1.934 0.053117 .  
## PC51        -3.713e-04  3.417e-04   -1.087 0.277221    
## PC52        -1.187e-03  3.420e-04   -3.471 0.000523 ***
## PC53        -3.275e-04  3.473e-04   -0.943 0.345749    
## PC54         1.245e-04  3.444e-04    0.362 0.717703    
## PC55         3.457e-04  3.452e-04    1.001 0.316696    
## PC56        -3.544e-04  3.524e-04   -1.006 0.314647    
## PC57         5.764e-04  3.521e-04    1.637 0.101624    
## PC58         1.039e-03  3.525e-04    2.949 0.003205 ** 
## PC59         3.031e-04  3.537e-04    0.857 0.391421    
## PC60         3.875e-05  3.529e-04    0.110 0.912564    
## PC61         8.183e-04  3.570e-04    2.292 0.021920 *  
## PC62         4.475e-04  3.574e-04    1.252 0.210534    
## PC63         6.753e-04  3.586e-04    1.883 0.059730 .  
## PC64         2.703e-04  3.582e-04    0.755 0.450398    
## PC65        -1.522e-04  3.632e-04   -0.419 0.675100    
## PC66         1.565e-04  3.592e-04    0.436 0.663142    
## PC67        -6.496e-04  3.653e-04   -1.778 0.075471 .  
## PC68        -2.493e-05  3.665e-04   -0.068 0.945777    
## PC69         1.045e-03  3.639e-04    2.871 0.004112 ** 
## PC70         3.648e-04  3.702e-04    0.985 0.324465    
## PC71        -5.113e-05  3.693e-04   -0.138 0.889889    
## PC72         9.559e-04  3.682e-04    2.596 0.009443 ** 
## PC73         5.030e-05  3.687e-04    0.136 0.891500    
## PC74         3.881e-04  3.714e-04    1.045 0.296052    
## PC75         4.399e-04  3.695e-04    1.191 0.233861    
## PC76        -3.644e-04  3.721e-04   -0.979 0.327459    
## PC77         1.107e-03  3.724e-04    2.974 0.002956 ** 
## PC78        -2.844e-04  3.719e-04   -0.765 0.444470    
## PC79        -1.018e-03  3.725e-04   -2.732 0.006310 ** 
## PC80        -3.998e-04  3.733e-04   -1.071 0.284132    
## PC81         3.806e-04  3.741e-04    1.017 0.309090    
## PC82         2.060e-05  3.729e-04    0.055 0.955949    
## PC83        -3.473e-04  3.759e-04   -0.924 0.355523    
## PC84         2.266e-04  3.767e-04    0.601 0.547566    
## PC85        -5.952e-04  3.766e-04   -1.580 0.114097    
## PC86        -3.743e-04  3.792e-04   -0.987 0.323762    
## PC87        -1.020e-03  3.763e-04   -2.710 0.006743 ** 
## PC88         2.179e-04  3.802e-04    0.573 0.566548    
## PC89         4.171e-04  3.786e-04    1.102 0.270690    
## PC90        -7.093e-04  3.788e-04   -1.873 0.061167 .  
## PC91         5.165e-04  3.839e-04    1.346 0.178512    
## PC92        -7.572e-05  3.813e-04   -0.199 0.842589    
## PC93         6.046e-04  3.829e-04    1.579 0.114442    
## PC94         1.522e-04  3.822e-04    0.398 0.690553    
## PC95         4.533e-04  3.808e-04    1.190 0.233979    
## PC96        -3.015e-04  3.813e-04   -0.791 0.429183    
## PC97        -1.845e-04  3.840e-04   -0.481 0.630840    
## PC98         6.540e-04  3.843e-04    1.702 0.088883 .  
## PC99        -1.421e-03  3.856e-04   -3.685 0.000231 ***
## PC100       -4.994e-04  3.868e-04   -1.291 0.196744    
## PC101        2.526e-04  3.852e-04    0.656 0.511993    
## PC102        3.120e-04  3.874e-04    0.805 0.420739    
## PC103       -5.112e-04  3.870e-04   -1.321 0.186582    
## PC104        1.592e-04  3.878e-04    0.411 0.681453    
## PC105       -4.513e-04  3.915e-04   -1.153 0.249061    
## PC106        2.192e-05  3.901e-04    0.056 0.955195    
## PC107       -1.273e-03  3.881e-04   -3.279 0.001049 ** 
## PC108        3.164e-04  3.886e-04    0.814 0.415539    
## PC109        4.245e-04  3.943e-04    1.077 0.281696    
## PC110       -4.415e-05  3.897e-04   -0.113 0.909790    
## PC111       -8.848e-04  3.910e-04   -2.263 0.023695 *  
## PC112       -1.163e-03  3.914e-04   -2.972 0.002970 ** 
## PC113        4.954e-04  3.904e-04    1.269 0.204514    
## PC114        7.547e-05  3.900e-04    0.194 0.846551    
## PC115       -8.165e-04  3.937e-04   -2.074 0.038115 *  
## PC116        1.341e-03  3.896e-04    3.441 0.000585 ***
## PC117        8.247e-04  3.942e-04    2.092 0.036468 *  
## PC118        9.646e-04  3.924e-04    2.458 0.013983 *  
## PC119       -3.786e-04  3.948e-04   -0.959 0.337689    
## PC120        5.851e-04  3.959e-04    1.478 0.139440    
## PC121       -2.337e-04  3.958e-04   -0.591 0.554853    
## PC122       -1.408e-03  3.927e-04   -3.585 0.000340 ***
## PC123        1.898e-04  3.950e-04    0.481 0.630879    
## PC124        1.607e-04  3.972e-04    0.405 0.685818    
## PC125       -6.586e-05  3.972e-04   -0.166 0.868323    
## PC126        9.596e-04  3.995e-04    2.402 0.016348 *  
## PC127       -1.066e-03  3.994e-04   -2.670 0.007604 ** 
## PC128       -1.616e-05  3.995e-04   -0.040 0.967739    
## PC129       -1.770e-04  3.985e-04   -0.444 0.656929    
## PC130        5.703e-04  4.009e-04    1.423 0.154934    
## PC131        6.614e-05  4.014e-04    0.165 0.869112    
## PC132       -1.563e-05  4.036e-04   -0.039 0.969113    
## PC133       -1.254e-03  4.005e-04   -3.132 0.001747 ** 
## PC134        6.838e-04  4.017e-04    1.702 0.088739 .  
## PC135       -9.845e-05  4.066e-04   -0.242 0.808714    
## PC136        1.243e-04  4.033e-04    0.308 0.757883    
## PC137       -8.539e-04  4.064e-04   -2.101 0.035700 *  
## PC138        1.573e-04  4.033e-04    0.390 0.696536    
## PC139       -7.689e-04  4.075e-04   -1.887 0.059201 .  
## PC140        3.405e-04  4.056e-04    0.839 0.401273    
## PC141       -1.335e-04  4.053e-04   -0.329 0.741839    
## PC142        1.956e-04  4.063e-04    0.481 0.630186    
## PC143        1.127e-03  4.059e-04    2.777 0.005498 ** 
## PC144        6.876e-04  4.090e-04    1.681 0.092825 .  
## PC145       -1.132e-04  4.089e-04   -0.277 0.781867    
## PC146        2.314e-03  4.096e-04    5.650 1.68e-08 ***
## PC147       -2.315e-04  4.085e-04   -0.567 0.570891    
## PC148       -4.911e-04  4.107e-04   -1.196 0.231792    
## PC149       -4.523e-05  4.096e-04   -0.110 0.912066    
## PC150       -5.764e-04  4.101e-04   -1.405 0.159946    
## PC151       -8.713e-05  4.108e-04   -0.212 0.832031    
## PC152       -1.260e-03  4.104e-04   -3.071 0.002143 ** 
## PC153       -6.404e-04  4.106e-04   -1.560 0.118934    
## PC154        7.080e-04  4.127e-04    1.716 0.086303 .  
## PC155       -5.064e-04  4.130e-04   -1.226 0.220216    
## PC156        3.395e-04  4.139e-04    0.820 0.412176    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.03225 on 5427 degrees of freedom
## Multiple R-squared:  0.2301, Adjusted R-squared:  0.2079 
## F-statistic:  10.4 on 156 and 5427 DF,  p-value: < 2.2e-16
cd.full = plot.diagnostics(model=model.full, train=data.train)

## [1] "Number of data points that have Cook's D > 4/n: 270"
## [1] "Number of data points that have Cook's D > 1: 0"

Checking with removal of high influence points

high.cd = names(cd.full[cd.full > 4/nrow(data.train)])

#save dataset with high.cd flagged
t = data.train %>% 
  rownames_to_column() %>%
  mutate(high.cd = ifelse(rowname %in% high.cd,1,0))
#write.csv(t,file='data_high_cd_flag.csv',row.names = F)
###
data.train2 = data.train[!(rownames(data.train)) %in% high.cd,]
model.full2 = lm(formula , data.train2)
summary(model.full2)
## 
## Call:
## lm(formula = formula, data = data.train2)
## 
## Residuals:
##       Min        1Q    Median        3Q       Max 
## -0.076433 -0.020875 -0.002849  0.019196  0.085203 
## 
## Coefficients:
##               Estimate Std. Error  t value Pr(>|t|)    
## (Intercept)  2.094e+00  3.952e-04 5298.868  < 2e-16 ***
## PC1          2.697e-04  2.660e-05   10.136  < 2e-16 ***
## PC2         -3.831e-04  3.141e-05  -12.199  < 2e-16 ***
## PC3         -1.364e-04  3.758e-05   -3.630 0.000286 ***
## PC4         -6.954e-05  3.854e-05   -1.804 0.071215 .  
## PC5         -2.346e-04  3.883e-05   -6.040 1.64e-09 ***
## PC6          8.715e-05  3.944e-05    2.210 0.027177 *  
## PC7         -1.779e-04  4.021e-05   -4.424 9.90e-06 ***
## PC8         -1.058e-04  4.136e-05   -2.558 0.010567 *  
## PC9          1.096e-04  4.285e-05    2.558 0.010558 *  
## PC10         2.573e-05  4.402e-05    0.585 0.558867    
## PC11         4.992e-04  4.558e-05   10.951  < 2e-16 ***
## PC12        -3.876e-04  4.597e-05   -8.430  < 2e-16 ***
## PC13         2.583e-04  4.748e-05    5.441 5.54e-08 ***
## PC14         6.212e-04  4.819e-05   12.892  < 2e-16 ***
## PC15        -1.272e-04  4.948e-05   -2.571 0.010175 *  
## PC16         1.897e-04  5.160e-05    3.676 0.000239 ***
## PC17        -6.545e-05  5.338e-05   -1.226 0.220173    
## PC18        -7.167e-05  5.569e-05   -1.287 0.198158    
## PC19        -1.125e-04  5.800e-05   -1.939 0.052566 .  
## PC20         4.207e-04  6.204e-05    6.781 1.32e-11 ***
## PC21        -2.845e-04  6.453e-05   -4.409 1.06e-05 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.02879 on 5292 degrees of freedom
## Multiple R-squared:  0.1335, Adjusted R-squared:  0.1301 
## F-statistic: 38.83 on 21 and 5292 DF,  p-value: < 2.2e-16
cd.full2 = plot.diagnostics(model.full2, data.train2)

## [1] "Number of data points that have Cook's D > 4/n: 202"
## [1] "Number of data points that have Cook's D > 1: 0"
# much more normal residuals than before. 
# Checking to see if distributions are different and if so whcih variables
# High Leverage Plot 
plotData = data.train %>% 
  rownames_to_column() %>%
  mutate(type=ifelse(rowname %in% high.cd,'High','Normal')) %>%
  dplyr::select(type,target=one_of(label.names))

ggplot(data=plotData, aes(x=type,y=target)) +
  geom_boxplot(fill='light blue',outlier.shape=NA) +
  scale_y_continuous(name="Target Variable Values",label=scales::comma_format(accuracy=.1)) +
  theme_light() +
  ggtitle('Distribution of High Leverage Points and Normal  Points')

# 2 sample t-tests

plotData = data.train %>% 
  rownames_to_column() %>%
  mutate(type=ifelse(rowname %in% high.cd,'High','Normal')) %>%
  dplyr::select(type,one_of(feature.names))

comp.test = lapply(dplyr::select(plotData, one_of(feature.names))
                   , function(x) t.test(x ~ plotData$type, var.equal = TRUE)) 

sig.comp = list.filter(comp.test, p.value < 0.05)
sapply(sig.comp, function(x) x[['p.value']])
##          PC1         PC14         PC83        PC104        PC112        PC114 
## 1.534155e-06 1.032959e-04 5.984026e-03 4.926227e-02 1.902915e-02 2.357616e-02
mm = melt(plotData, id=c('type')) %>% filter(variable %in% names(sig.comp))

ggplot(mm,aes(x=type, y=value)) +
  geom_boxplot()+
  facet_wrap(~variable, ncol=5, scales = 'free_y') +
  scale_y_continuous(name="values",label=scales::comma_format(accuracy=.1)) +
  ggtitle('Distribution of High Leverage Points and Normal Points')

# Distribution (box) Plots
mm = melt(plotData, id=c('type'))

ggplot(mm,aes(x=type, y=value)) +
  geom_boxplot()+
  facet_wrap(~variable, ncol=8, scales = 'free_y') +
  scale_y_continuous(name="values",label=scales::comma_format(accuracy=.1)) +
  ggtitle('Distribution of High Leverage Points and Normal Points')

Grand Means Model

model.null = lm(grand.mean.formula, data.train)
summary(model.null)
## 
## Call:
## lm(formula = grand.mean.formula, data = data.train)
## 
## Residuals:
##       Min        1Q    Median        3Q       Max 
## -0.114676 -0.023705 -0.003387  0.020847  0.190636 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept) 2.096552   0.000485    4323   <2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.03624 on 5583 degrees of freedom

Variable Selection

Basic: http://www.stat.columbia.edu/~martin/W2024/R10.pdf Cross Validation + Other Metrics: http://www.sthda.com/english/articles/37-model-selection-essentials-in-r/154-stepwise-regression-essentials-in-r/

Forward Selection with CV

Train

if (algo.forward.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   , data = data.train
                                   , method = "leapForward"
                                   , feature.names = feature.names)
  model.forward = returned$model
  id = returned$id
}
## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 60 on full training set
## [1] "All models results"
##     nvmax       RMSE   Rsquared        MAE       RMSESD RsquaredSD        MAESD
## 1       1 0.03460715 0.08910629 0.02686885 0.0011220847 0.02445943 0.0006474464
## 2       2 0.03427537 0.10669480 0.02663179 0.0010083889 0.02883392 0.0005758770
## 3       3 0.03411994 0.11496880 0.02653867 0.0010265906 0.03075293 0.0006063719
## 4       4 0.03379615 0.13189897 0.02630712 0.0010207545 0.03344351 0.0005144619
## 5       5 0.03350142 0.14741335 0.02600541 0.0011291975 0.04089043 0.0006240441
## 6       6 0.03335616 0.15492598 0.02585664 0.0011817548 0.04260564 0.0006431018
## 7       7 0.03334671 0.15524765 0.02588207 0.0011918151 0.04207618 0.0006350839
## 8       8 0.03331117 0.15702802 0.02587148 0.0011827844 0.04160488 0.0006327650
## 9       9 0.03322305 0.16163226 0.02580335 0.0012175344 0.04406315 0.0006238541
## 10     10 0.03304920 0.17039248 0.02567687 0.0012542700 0.04567367 0.0006956817
## 11     11 0.03294153 0.17540794 0.02559498 0.0012363522 0.04359549 0.0007022903
## 12     12 0.03302015 0.17164164 0.02563644 0.0012557230 0.04412034 0.0006919223
## 13     13 0.03305043 0.17016840 0.02564263 0.0012468449 0.04189529 0.0007132106
## 14     14 0.03301946 0.17187785 0.02562588 0.0012420740 0.04196257 0.0006972580
## 15     15 0.03304794 0.17044746 0.02565163 0.0012271977 0.04033782 0.0006823314
## 16     16 0.03304618 0.17060985 0.02566269 0.0012216908 0.04034003 0.0006564886
## 17     17 0.03302631 0.17169036 0.02566679 0.0012273896 0.04149103 0.0006758318
## 18     18 0.03299358 0.17333306 0.02564312 0.0012334455 0.04215032 0.0006963940
## 19     19 0.03296701 0.17451230 0.02564028 0.0012021627 0.04090036 0.0006805821
## 20     20 0.03295161 0.17520514 0.02562301 0.0011632949 0.03937627 0.0006490421
## 21     21 0.03294342 0.17569547 0.02561491 0.0011339592 0.03845149 0.0006403542
## 22     22 0.03296495 0.17490667 0.02562498 0.0011555594 0.03944700 0.0006626812
## 23     23 0.03294635 0.17573083 0.02558977 0.0011508121 0.03910113 0.0006454214
## 24     24 0.03293832 0.17614576 0.02559123 0.0011700580 0.03838173 0.0006842952
## 25     25 0.03292094 0.17695723 0.02557273 0.0011747830 0.03836036 0.0006890772
## 26     26 0.03287875 0.17905169 0.02552170 0.0011784192 0.03870279 0.0006941080
## 27     27 0.03288472 0.17889913 0.02553718 0.0011702651 0.03954319 0.0006737343
## 28     28 0.03285560 0.18010664 0.02550215 0.0011583478 0.03826274 0.0006693062
## 29     29 0.03285475 0.18020821 0.02548120 0.0011296991 0.03726967 0.0006298356
## 30     30 0.03281703 0.18202381 0.02546579 0.0011283144 0.03763312 0.0006352938
## 31     31 0.03280656 0.18265860 0.02545133 0.0011510835 0.03825756 0.0006202786
## 32     32 0.03280791 0.18263516 0.02544072 0.0011443911 0.03720283 0.0006090150
## 33     33 0.03279398 0.18328647 0.02543037 0.0011243804 0.03696881 0.0005880315
## 34     34 0.03277028 0.18445467 0.02540484 0.0011235998 0.03730505 0.0006068126
## 35     35 0.03274883 0.18530440 0.02539191 0.0011135069 0.03586909 0.0006076907
## 36     36 0.03274507 0.18551295 0.02539207 0.0010892369 0.03574803 0.0006177277
## 37     37 0.03273283 0.18596732 0.02538464 0.0010481181 0.03410175 0.0005812428
## 38     38 0.03271812 0.18664667 0.02536959 0.0010337151 0.03370093 0.0005685931
## 39     39 0.03274129 0.18560255 0.02539236 0.0010345439 0.03348123 0.0005586956
## 40     40 0.03273552 0.18591130 0.02539526 0.0010355019 0.03309684 0.0005569333
## 41     41 0.03272108 0.18657733 0.02537083 0.0010303174 0.03236046 0.0005471613
## 42     42 0.03271963 0.18666364 0.02537446 0.0010197381 0.03196622 0.0005381671
## 43     43 0.03272124 0.18654185 0.02538236 0.0009911899 0.03159428 0.0005331453
## 44     44 0.03274289 0.18567679 0.02539750 0.0009951781 0.03234625 0.0005310049
## 45     45 0.03275325 0.18539350 0.02541582 0.0010009944 0.03346403 0.0005461326
## 46     46 0.03275542 0.18529239 0.02541128 0.0009914222 0.03250482 0.0005351964
## 47     47 0.03276933 0.18460853 0.02543274 0.0009681256 0.03126429 0.0005136176
## 48     48 0.03274988 0.18562391 0.02541908 0.0009804754 0.03176915 0.0005148088
## 49     49 0.03275553 0.18549823 0.02542525 0.0010007573 0.03211922 0.0005294594
## 50     50 0.03277106 0.18485168 0.02543581 0.0010039392 0.03198443 0.0005320075
## 51     51 0.03277763 0.18450302 0.02543837 0.0009952213 0.03095233 0.0005173043
## 52     52 0.03278798 0.18409848 0.02545199 0.0009908014 0.03089017 0.0005212592
## 53     53 0.03277648 0.18467673 0.02543358 0.0010009955 0.03093738 0.0005100227
## 54     54 0.03276073 0.18545779 0.02541901 0.0009984671 0.03126142 0.0005210751
## 55     55 0.03274715 0.18616129 0.02541850 0.0010127485 0.03203220 0.0005248897
## 56     56 0.03273869 0.18660366 0.02540922 0.0010352073 0.03215573 0.0005547967
## 57     57 0.03270805 0.18803081 0.02539284 0.0010466131 0.03236273 0.0005550320
## 58     58 0.03270979 0.18808059 0.02539751 0.0010551510 0.03266562 0.0005609570
## 59     59 0.03271178 0.18802159 0.02539465 0.0010524592 0.03227840 0.0005608017
## 60     60 0.03269844 0.18866632 0.02538764 0.0010550590 0.03309063 0.0005611154
## 61     61 0.03270491 0.18837209 0.02539294 0.0010625515 0.03314899 0.0005596570
## 62     62 0.03272511 0.18746767 0.02540607 0.0010577008 0.03277498 0.0005637775
## 63     63 0.03272027 0.18765640 0.02540109 0.0010545200 0.03269457 0.0005680688
## 64     64 0.03270861 0.18823243 0.02538868 0.0010488503 0.03268527 0.0005743375
## 65     65 0.03271591 0.18795256 0.02539107 0.0010446916 0.03285781 0.0005841722
## 66     66 0.03271996 0.18775080 0.02539361 0.0010436172 0.03313321 0.0005918850
## 67     67 0.03271843 0.18788949 0.02539473 0.0010417985 0.03311181 0.0005890268
## 68     68 0.03271644 0.18803325 0.02539528 0.0010572510 0.03367356 0.0006088207
## 69     69 0.03272182 0.18777941 0.02540272 0.0010492718 0.03359551 0.0005994725
## 70     70 0.03272822 0.18752142 0.02539644 0.0010454808 0.03360970 0.0006019131
## 71     71 0.03273484 0.18726898 0.02539376 0.0010367535 0.03347874 0.0006027592
## 72     72 0.03273453 0.18725948 0.02538632 0.0010391762 0.03314719 0.0006078312
## 73     73 0.03272984 0.18750358 0.02538296 0.0010293007 0.03315450 0.0006026834
## 74     74 0.03273001 0.18755417 0.02537867 0.0010220918 0.03373738 0.0006058184
## 75     75 0.03272151 0.18796897 0.02536932 0.0010207351 0.03387082 0.0006093350
## 76     76 0.03273155 0.18750763 0.02537749 0.0010264015 0.03372761 0.0006173888
## 77     77 0.03273199 0.18745650 0.02538229 0.0010206493 0.03335904 0.0006102604
## 78     78 0.03274348 0.18701344 0.02539033 0.0010311402 0.03405252 0.0006209184
## 79     79 0.03274018 0.18716972 0.02538595 0.0010328567 0.03409210 0.0006250520
## 80     80 0.03273313 0.18753142 0.02538325 0.0010288814 0.03415561 0.0006183714
## 81     81 0.03273702 0.18735078 0.02538304 0.0010282931 0.03407795 0.0006118771
## 82     82 0.03274299 0.18714531 0.02539701 0.0010228526 0.03416576 0.0006069899
## 83     83 0.03275191 0.18674431 0.02540849 0.0010168319 0.03429298 0.0006018370
## 84     84 0.03275533 0.18661884 0.02541683 0.0010305542 0.03445081 0.0006140984
## 85     85 0.03276109 0.18637832 0.02542296 0.0010319170 0.03436306 0.0006123312
## 86     86 0.03275271 0.18676943 0.02541477 0.0010365623 0.03448032 0.0006130590
## 87     87 0.03275085 0.18683660 0.02540482 0.0010329945 0.03391209 0.0006082194
## 88     88 0.03275558 0.18659036 0.02541449 0.0010249052 0.03350529 0.0005980305
## 89     89 0.03275130 0.18680618 0.02540771 0.0010225812 0.03349312 0.0005912777
## 90     90 0.03274594 0.18704682 0.02540109 0.0010197449 0.03359319 0.0005842610
## 91     91 0.03275006 0.18688455 0.02540132 0.0010248638 0.03339890 0.0005852373
## 92     92 0.03275331 0.18673982 0.02540381 0.0010204365 0.03325324 0.0005836473
## 93     93 0.03275648 0.18660385 0.02540999 0.0010238662 0.03333798 0.0005899686
## 94     94 0.03274958 0.18692970 0.02540860 0.0010116514 0.03317192 0.0005919200
## 95     95 0.03273868 0.18744788 0.02539715 0.0010163919 0.03315470 0.0005976378
## 96     96 0.03273149 0.18779947 0.02539344 0.0010166997 0.03328330 0.0006008236
## 97     97 0.03273029 0.18786826 0.02539032 0.0010244333 0.03357123 0.0006056640
## 98     98 0.03273080 0.18787056 0.02539207 0.0010104930 0.03358452 0.0005958696
## 99     99 0.03272810 0.18803368 0.02539175 0.0010137891 0.03378287 0.0005986268
## 100   100 0.03272301 0.18824349 0.02539125 0.0010206948 0.03381989 0.0005978662
## 101   101 0.03272920 0.18795645 0.02539981 0.0010240915 0.03386609 0.0005944838
## 102   102 0.03272849 0.18801247 0.02539960 0.0010276786 0.03380193 0.0005966001
## 103   103 0.03272510 0.18816484 0.02539236 0.0010257268 0.03375440 0.0005951537
## 104   104 0.03273079 0.18797972 0.02540040 0.0010325908 0.03401229 0.0006042888
## 105   105 0.03272391 0.18831989 0.02539238 0.0010324947 0.03421325 0.0006057019
## 106   106 0.03273503 0.18787238 0.02540413 0.0010377774 0.03442339 0.0006134727
## 107   107 0.03273528 0.18787285 0.02540490 0.0010400810 0.03430067 0.0006150265
## 108   108 0.03272783 0.18821344 0.02539738 0.0010422622 0.03432022 0.0006228197
## 109   109 0.03272983 0.18811025 0.02539221 0.0010406590 0.03413780 0.0006227490
## 110   110 0.03273085 0.18807574 0.02538730 0.0010418965 0.03392532 0.0006195427
## 111   111 0.03273212 0.18800040 0.02538795 0.0010329692 0.03364180 0.0006130944
## 112   112 0.03273683 0.18775618 0.02538907 0.0010264433 0.03347160 0.0006038173
## 113   113 0.03273756 0.18775911 0.02538796 0.0010275569 0.03374325 0.0006030993
## 114   114 0.03274041 0.18761830 0.02538593 0.0010270854 0.03368049 0.0006029527
## 115   115 0.03274551 0.18738119 0.02538824 0.0010294860 0.03345724 0.0006024729
## 116   116 0.03274389 0.18744889 0.02538653 0.0010323134 0.03328277 0.0006058579
## 117   117 0.03274602 0.18735540 0.02539042 0.0010277931 0.03332876 0.0005996376
## 118   118 0.03274361 0.18748651 0.02538897 0.0010279936 0.03369465 0.0006009555
## 119   119 0.03273670 0.18780619 0.02538066 0.0010291764 0.03396010 0.0006065886
## 120   120 0.03273896 0.18770059 0.02538110 0.0010312515 0.03376811 0.0006032755
## 121   121 0.03273789 0.18774169 0.02537867 0.0010366808 0.03375552 0.0006053470
## 122   122 0.03273628 0.18783880 0.02537426 0.0010359416 0.03395179 0.0006074492
## 123   123 0.03273583 0.18786227 0.02537592 0.0010390961 0.03391936 0.0006073419
## 124   124 0.03273042 0.18809748 0.02537206 0.0010363843 0.03393317 0.0006048800
## 125   125 0.03273460 0.18790452 0.02537577 0.0010340742 0.03375031 0.0006041654
## 126   126 0.03273081 0.18807814 0.02537257 0.0010313721 0.03379566 0.0006016746
## 127   127 0.03273348 0.18795871 0.02537387 0.0010304346 0.03391211 0.0006032236
## 128   128 0.03273482 0.18789408 0.02537281 0.0010306645 0.03393882 0.0006056804
## 129   129 0.03273502 0.18789667 0.02537506 0.0010317609 0.03392485 0.0006048815
## 130   130 0.03273317 0.18799893 0.02537240 0.0010349140 0.03409805 0.0006096305
## 131   131 0.03273157 0.18808191 0.02536922 0.0010360207 0.03425999 0.0006123594
## 132   132 0.03272728 0.18826976 0.02536604 0.0010376921 0.03440363 0.0006137921
## 133   133 0.03272652 0.18829927 0.02536554 0.0010345628 0.03436332 0.0006095151
## 134   134 0.03272717 0.18824400 0.02536648 0.0010316031 0.03413291 0.0006063551
## 135   135 0.03272861 0.18818103 0.02536766 0.0010319885 0.03406399 0.0006070329
## 136   136 0.03272840 0.18817615 0.02536682 0.0010306691 0.03391950 0.0006046425
## 137   137 0.03272914 0.18815963 0.02536852 0.0010303920 0.03395362 0.0006029156
## 138   138 0.03272688 0.18825322 0.02536591 0.0010303373 0.03390046 0.0006014799
## 139   139 0.03272841 0.18818724 0.02536586 0.0010287661 0.03384073 0.0005987557
## 140   140 0.03272766 0.18822236 0.02536378 0.0010283025 0.03386597 0.0005988105
## 141   141 0.03272988 0.18812177 0.02536553 0.0010277518 0.03388808 0.0005997787
## 142   142 0.03272945 0.18813972 0.02536681 0.0010285129 0.03389507 0.0005996815
## 143   143 0.03273172 0.18803401 0.02536858 0.0010271318 0.03379862 0.0005983827
## 144   144 0.03273118 0.18805575 0.02536741 0.0010272286 0.03379135 0.0005984701
## 145   145 0.03273142 0.18803956 0.02536675 0.0010269212 0.03380784 0.0005989952
## 146   146 0.03273038 0.18808519 0.02536570 0.0010274374 0.03380405 0.0006001165
## 147   147 0.03273117 0.18805072 0.02536742 0.0010282407 0.03378384 0.0006002984
## 148   148 0.03273148 0.18803306 0.02536851 0.0010276387 0.03379042 0.0006005813
## 149   149 0.03273186 0.18801995 0.02536907 0.0010268925 0.03377052 0.0006005685
## 150   150 0.03273175 0.18802377 0.02536901 0.0010264534 0.03376062 0.0005998615
## 151   151 0.03273158 0.18802911 0.02536848 0.0010261612 0.03374164 0.0006001690
## 152   152 0.03273137 0.18803677 0.02536861 0.0010261549 0.03374407 0.0005998615
## 153   153 0.03273170 0.18802270 0.02536903 0.0010260366 0.03374201 0.0006000006
## 154   154 0.03273175 0.18801994 0.02536914 0.0010263029 0.03375111 0.0006003751
## 155   155 0.03273174 0.18801812 0.02536919 0.0010261404 0.03374596 0.0006003774
## 156   156 0.03273170 0.18801970 0.02536912 0.0010260694 0.03374294 0.0006003263
## [1] "Best Model"
##    nvmax
## 60    60

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  2.096734e+00  2.095889e+00  2.097578e+00
## PC1          2.864714e-04  2.298585e-04  3.430843e-04
## PC2         -3.807254e-04 -4.475032e-04 -3.139476e-04
## PC3         -8.108879e-05 -1.610158e-04 -1.161790e-06
## PC4         -8.943715e-05 -1.717642e-04 -7.110109e-06
## PC5         -2.377410e-04 -3.203356e-04 -1.551464e-04
## PC6          1.066122e-04  2.221774e-05  1.910067e-04
## PC7         -1.681047e-04 -2.536784e-04 -8.253090e-05
## PC8         -1.304845e-04 -2.183161e-04 -4.265283e-05
## PC9          1.005409e-04  9.396046e-06  1.916858e-04
## PC11         4.782470e-04  3.808854e-04  5.756087e-04
## PC12        -3.662753e-04 -4.641971e-04 -2.683536e-04
## PC13         3.070715e-04  2.060052e-04  4.081378e-04
## PC14         5.442716e-04  4.419095e-04  6.466337e-04
## PC15        -1.126470e-04 -2.179876e-04 -7.306318e-06
## PC16         2.139724e-04  1.042408e-04  3.237040e-04
## PC20         4.000364e-04  2.677662e-04  5.323067e-04
## PC21        -3.549027e-04 -4.930481e-04 -2.167573e-04
## PC22         1.860156e-03  1.714040e-03  2.006272e-03
## PC23        -5.033151e-04 -9.217876e-04 -8.484262e-05
## PC25         4.304268e-04 -3.069138e-05  8.915450e-04
## PC37         6.740412e-04  5.728323e-05  1.290799e-03
## PC39        -7.536803e-04 -1.373151e-03 -1.342099e-04
## PC47        -6.216417e-04 -1.265883e-03  2.260009e-05
## PC50         6.637683e-04  6.747998e-06  1.320789e-03
## PC52        -1.184340e-03 -1.851822e-03 -5.168588e-04
## PC57         5.712924e-04 -1.157284e-04  1.258313e-03
## PC58         1.048528e-03  3.606851e-04  1.736372e-03
## PC61         8.429992e-04  1.466646e-04  1.539334e-03
## PC63         6.993862e-04 -1.444877e-07  1.398917e-03
## PC67        -6.397041e-04 -1.352111e-03  7.270237e-05
## PC69         1.044065e-03  3.339378e-04  1.754192e-03
## PC72         9.661565e-04  2.474483e-04  1.684865e-03
## PC77         1.105688e-03  3.790291e-04  1.832347e-03
## PC79        -1.009839e-03 -1.736829e-03 -2.828478e-04
## PC85        -5.895932e-04 -1.324399e-03  1.452122e-04
## PC87        -1.047616e-03 -1.782099e-03 -3.131339e-04
## PC90        -6.935265e-04 -1.432596e-03  4.554267e-05
## PC93         6.296589e-04 -1.177987e-04  1.377116e-03
## PC98         6.428428e-04 -1.070484e-04  1.392734e-03
## PC99        -1.363831e-03 -2.115953e-03 -6.117087e-04
## PC107       -1.270600e-03 -2.028160e-03 -5.130396e-04
## PC111       -8.857512e-04 -1.648669e-03 -1.228339e-04
## PC112       -1.168778e-03 -1.932519e-03 -4.050357e-04
## PC115       -8.263315e-04 -1.594540e-03 -5.812314e-05
## PC116        1.385187e-03  6.246659e-04  2.145707e-03
## PC117        8.163043e-04  4.737984e-05  1.585229e-03
## PC118        9.866072e-04  2.207404e-04  1.752474e-03
## PC122       -1.390951e-03 -2.157106e-03 -6.247957e-04
## PC126        9.561386e-04  1.766049e-04  1.735672e-03
## PC127       -1.070588e-03 -1.850154e-03 -2.910221e-04
## PC133       -1.214145e-03 -1.995517e-03 -4.327725e-04
## PC134        6.879742e-04 -9.600879e-05  1.471957e-03
## PC137       -8.324059e-04 -1.625185e-03 -3.962735e-05
## PC139       -7.629960e-04 -1.558015e-03  3.202295e-05
## PC143        1.136332e-03  3.444744e-04  1.928190e-03
## PC144        6.994141e-04 -9.858229e-05  1.497410e-03
## PC146        2.334270e-03  1.535459e-03  3.133081e-03
## PC152       -1.296909e-03 -2.097718e-03 -4.961007e-04
## PC153       -6.618241e-04 -1.462808e-03  1.391600e-04
## PC154        7.113556e-04 -9.418126e-05  1.516892e-03

Test

if (algo.forward.caret == TRUE){
    test.model(model=model.forward, test=data.test
             ,method = 'leapForward',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.034   2.085   2.097   2.097   2.110   2.148 
## [1] "leapForward  Test MSE: 0.00105127042865879"

Backward Elimination with CV

Train

if (algo.backward.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "leapBackward"
                                   ,feature.names =  feature.names)
  model.backward = returned$model
  id = returned$id
}
## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 60 on full training set
## [1] "All models results"
##     nvmax       RMSE   Rsquared        MAE       RMSESD RsquaredSD        MAESD
## 1       1 0.03460715 0.08910629 0.02686885 0.0011220847 0.02445943 0.0006474464
## 2       2 0.03427537 0.10669480 0.02663179 0.0010083889 0.02883392 0.0005758770
## 3       3 0.03411994 0.11496880 0.02653867 0.0010265906 0.03075293 0.0006063719
## 4       4 0.03379615 0.13189897 0.02630712 0.0010207545 0.03344351 0.0005144619
## 5       5 0.03350142 0.14741335 0.02600541 0.0011291975 0.04089043 0.0006240441
## 6       6 0.03335616 0.15492598 0.02585664 0.0011817548 0.04260564 0.0006431018
## 7       7 0.03334671 0.15524765 0.02588207 0.0011918151 0.04207618 0.0006350839
## 8       8 0.03331117 0.15702802 0.02587148 0.0011827844 0.04160488 0.0006327650
## 9       9 0.03322305 0.16163226 0.02580335 0.0012175344 0.04406315 0.0006238541
## 10     10 0.03304920 0.17039248 0.02567687 0.0012542700 0.04567367 0.0006956817
## 11     11 0.03294153 0.17540794 0.02559498 0.0012363522 0.04359549 0.0007022903
## 12     12 0.03302015 0.17164164 0.02563644 0.0012557230 0.04412034 0.0006919223
## 13     13 0.03305043 0.17016840 0.02564263 0.0012468449 0.04189529 0.0007132106
## 14     14 0.03301946 0.17187785 0.02562588 0.0012420740 0.04196257 0.0006972580
## 15     15 0.03304794 0.17044746 0.02565163 0.0012271977 0.04033782 0.0006823314
## 16     16 0.03304618 0.17060985 0.02566269 0.0012216908 0.04034003 0.0006564886
## 17     17 0.03302631 0.17169036 0.02566679 0.0012273896 0.04149103 0.0006758318
## 18     18 0.03299358 0.17333306 0.02564312 0.0012334455 0.04215032 0.0006963940
## 19     19 0.03296701 0.17451230 0.02564028 0.0012021627 0.04090036 0.0006805821
## 20     20 0.03295161 0.17520514 0.02562301 0.0011632949 0.03937627 0.0006490421
## 21     21 0.03294342 0.17569547 0.02561491 0.0011339592 0.03845149 0.0006403542
## 22     22 0.03296495 0.17490667 0.02562498 0.0011555594 0.03944700 0.0006626812
## 23     23 0.03294635 0.17573083 0.02558977 0.0011508121 0.03910113 0.0006454214
## 24     24 0.03293715 0.17617527 0.02558442 0.0011674877 0.03834398 0.0006736806
## 25     25 0.03291930 0.17704289 0.02556458 0.0011711946 0.03825170 0.0006768927
## 26     26 0.03287788 0.17907243 0.02551585 0.0011765267 0.03867622 0.0006855546
## 27     27 0.03288472 0.17889913 0.02553718 0.0011702651 0.03954319 0.0006737343
## 28     28 0.03285560 0.18010664 0.02550215 0.0011583478 0.03826274 0.0006693062
## 29     29 0.03284185 0.18091676 0.02547300 0.0011530386 0.03879046 0.0006477823
## 30     30 0.03280537 0.18266975 0.02546378 0.0011495330 0.03901952 0.0006397595
## 31     31 0.03279901 0.18309691 0.02544541 0.0011646416 0.03919244 0.0006336598
## 32     32 0.03280638 0.18267257 0.02544309 0.0011471350 0.03728355 0.0006035704
## 33     33 0.03279398 0.18328647 0.02543037 0.0011243804 0.03696881 0.0005880315
## 34     34 0.03277028 0.18445467 0.02540484 0.0011235998 0.03730505 0.0006068126
## 35     35 0.03274883 0.18530440 0.02539191 0.0011135069 0.03586909 0.0006076907
## 36     36 0.03274507 0.18551295 0.02539207 0.0010892369 0.03574803 0.0006177277
## 37     37 0.03273283 0.18596732 0.02538464 0.0010481181 0.03410175 0.0005812428
## 38     38 0.03271812 0.18664667 0.02536959 0.0010337151 0.03370093 0.0005685931
## 39     39 0.03273822 0.18574310 0.02539413 0.0010344984 0.03346221 0.0005584500
## 40     40 0.03274135 0.18566898 0.02539934 0.0010299346 0.03326166 0.0005560800
## 41     41 0.03274131 0.18566491 0.02539085 0.0010216059 0.03263768 0.0005427220
## 42     42 0.03274178 0.18566328 0.02539768 0.0010029839 0.03240042 0.0005330497
## 43     43 0.03272714 0.18627988 0.02537984 0.0009931485 0.03176621 0.0005296900
## 44     44 0.03274908 0.18541148 0.02539573 0.0009972581 0.03252447 0.0005276035
## 45     45 0.03275325 0.18539350 0.02541582 0.0010009944 0.03346403 0.0005461326
## 46     46 0.03275542 0.18529239 0.02541128 0.0009914222 0.03250482 0.0005351964
## 47     47 0.03276933 0.18460853 0.02543274 0.0009681256 0.03126429 0.0005136176
## 48     48 0.03274988 0.18562391 0.02541908 0.0009804754 0.03176915 0.0005148088
## 49     49 0.03275553 0.18549823 0.02542525 0.0010007573 0.03211922 0.0005294594
## 50     50 0.03276518 0.18508657 0.02543262 0.0009898485 0.03169052 0.0005268329
## 51     51 0.03277534 0.18456973 0.02544177 0.0009896558 0.03086487 0.0005231317
## 52     52 0.03278702 0.18411787 0.02545322 0.0009884840 0.03086504 0.0005233604
## 53     53 0.03277158 0.18482715 0.02543859 0.0009888290 0.03073065 0.0005190932
## 54     54 0.03275569 0.18567027 0.02542204 0.0010037024 0.03145190 0.0005342170
## 55     55 0.03274858 0.18610955 0.02542327 0.0010200009 0.03227789 0.0005288334
## 56     56 0.03273439 0.18676675 0.02540536 0.0010313737 0.03210915 0.0005412121
## 57     57 0.03270573 0.18813669 0.02539744 0.0010476874 0.03240237 0.0005572162
## 58     58 0.03270979 0.18808059 0.02539751 0.0010551510 0.03266562 0.0005609570
## 59     59 0.03271178 0.18802159 0.02539465 0.0010524592 0.03227840 0.0005608017
## 60     60 0.03269266 0.18893497 0.02538453 0.0010585926 0.03303490 0.0005627345
## 61     61 0.03270653 0.18832407 0.02539257 0.0010603466 0.03324198 0.0005581100
## 62     62 0.03272789 0.18736542 0.02540479 0.0010550276 0.03285952 0.0005624865
## 63     63 0.03272030 0.18765641 0.02540052 0.0010545069 0.03269457 0.0005683311
## 64     64 0.03270704 0.18832722 0.02538969 0.0010477951 0.03282912 0.0005747758
## 65     65 0.03270227 0.18861382 0.02538261 0.0010417427 0.03304326 0.0005818183
## 66     66 0.03269917 0.18874452 0.02537871 0.0010456097 0.03351914 0.0005984833
## 67     67 0.03270317 0.18855892 0.02538448 0.0010334160 0.03302725 0.0005881168
## 68     68 0.03270527 0.18850237 0.02538364 0.0010441783 0.03354538 0.0006071791
## 69     69 0.03272367 0.18768989 0.02540209 0.0010475336 0.03348799 0.0006005127
## 70     70 0.03272955 0.18745657 0.02539438 0.0010442300 0.03353255 0.0006052834
## 71     71 0.03273484 0.18726898 0.02539376 0.0010367535 0.03347874 0.0006027592
## 72     72 0.03273453 0.18725948 0.02538632 0.0010391762 0.03314719 0.0006078312
## 73     73 0.03272839 0.18756128 0.02538198 0.0010300782 0.03318120 0.0006021088
## 74     74 0.03272702 0.18766547 0.02537814 0.0010236755 0.03378648 0.0006054991
## 75     75 0.03272151 0.18796897 0.02536932 0.0010207351 0.03387082 0.0006093350
## 76     76 0.03273155 0.18750763 0.02537749 0.0010264015 0.03372761 0.0006173888
## 77     77 0.03273453 0.18733429 0.02538198 0.0010197202 0.03342693 0.0006103005
## 78     78 0.03274608 0.18688896 0.02539000 0.0010301481 0.03411692 0.0006209598
## 79     79 0.03274246 0.18706965 0.02538996 0.0010313375 0.03417372 0.0006254173
## 80     80 0.03273985 0.18724204 0.02538688 0.0010272337 0.03417139 0.0006205032
## 81     81 0.03274295 0.18710479 0.02538757 0.0010274180 0.03404998 0.0006134103
## 82     82 0.03274165 0.18720303 0.02539787 0.0010220478 0.03427418 0.0006063048
## 83     83 0.03275302 0.18671987 0.02541056 0.0010176765 0.03432553 0.0006038851
## 84     84 0.03275669 0.18658343 0.02541857 0.0010315616 0.03449735 0.0006157346
## 85     85 0.03276244 0.18634348 0.02542515 0.0010328783 0.03440773 0.0006142841
## 86     86 0.03275300 0.18675775 0.02541514 0.0010367721 0.03449552 0.0006133972
## 87     87 0.03275085 0.18683660 0.02540482 0.0010329945 0.03391209 0.0006082194
## 88     88 0.03275558 0.18659036 0.02541449 0.0010249052 0.03350529 0.0005980305
## 89     89 0.03274938 0.18687668 0.02540380 0.0010179384 0.03340387 0.0005847973
## 90     90 0.03274650 0.18702034 0.02540064 0.0010210869 0.03362662 0.0005835311
## 91     91 0.03275006 0.18688455 0.02540132 0.0010248638 0.03339890 0.0005852373
## 92     92 0.03275331 0.18673982 0.02540381 0.0010204365 0.03325324 0.0005836473
## 93     93 0.03275648 0.18660385 0.02540999 0.0010238662 0.03333798 0.0005899686
## 94     94 0.03274958 0.18692970 0.02540860 0.0010116514 0.03317192 0.0005919200
## 95     95 0.03273868 0.18744788 0.02539715 0.0010163919 0.03315470 0.0005976378
## 96     96 0.03273448 0.18765647 0.02539152 0.0010201272 0.03332907 0.0006020619
## 97     97 0.03273178 0.18779492 0.02538923 0.0010230580 0.03338110 0.0006027204
## 98     98 0.03273268 0.18778564 0.02538895 0.0010095769 0.03344484 0.0005937327
## 99     99 0.03272935 0.18799604 0.02539432 0.0010135720 0.03379191 0.0005988461
## 100   100 0.03272615 0.18810639 0.02539415 0.0010192672 0.03391352 0.0005980475
## 101   101 0.03273274 0.18780812 0.02540364 0.0010215282 0.03401060 0.0005950691
## 102   102 0.03272588 0.18812232 0.02539921 0.0010260384 0.03387407 0.0005980094
## 103   103 0.03272704 0.18807822 0.02539417 0.0010290160 0.03385211 0.0005960393
## 104   104 0.03273079 0.18797972 0.02540040 0.0010325908 0.03401229 0.0006042888
## 105   105 0.03272391 0.18831989 0.02539238 0.0010324947 0.03421325 0.0006057019
## 106   106 0.03273503 0.18787238 0.02540413 0.0010377774 0.03442339 0.0006134727
## 107   107 0.03273528 0.18787285 0.02540490 0.0010400810 0.03430067 0.0006150265
## 108   108 0.03272783 0.18821344 0.02539738 0.0010422622 0.03432022 0.0006228197
## 109   109 0.03272983 0.18811025 0.02539221 0.0010406590 0.03413780 0.0006227490
## 110   110 0.03273085 0.18807574 0.02538730 0.0010418965 0.03392532 0.0006195427
## 111   111 0.03273212 0.18800040 0.02538795 0.0010329692 0.03364180 0.0006130944
## 112   112 0.03273683 0.18775618 0.02538907 0.0010264433 0.03347160 0.0006038173
## 113   113 0.03273756 0.18775911 0.02538796 0.0010275569 0.03374325 0.0006030993
## 114   114 0.03274041 0.18761830 0.02538593 0.0010270854 0.03368049 0.0006029527
## 115   115 0.03274551 0.18738119 0.02538824 0.0010294860 0.03345724 0.0006024729
## 116   116 0.03274389 0.18744889 0.02538653 0.0010323134 0.03328277 0.0006058579
## 117   117 0.03274602 0.18735540 0.02539042 0.0010277931 0.03332876 0.0005996376
## 118   118 0.03274361 0.18748651 0.02538897 0.0010279936 0.03369465 0.0006009555
## 119   119 0.03273670 0.18780619 0.02538066 0.0010291764 0.03396010 0.0006065886
## 120   120 0.03273896 0.18770059 0.02538110 0.0010312515 0.03376811 0.0006032755
## 121   121 0.03273789 0.18774169 0.02537867 0.0010366808 0.03375552 0.0006053470
## 122   122 0.03273628 0.18783880 0.02537426 0.0010359416 0.03395179 0.0006074492
## 123   123 0.03273755 0.18779303 0.02537801 0.0010382534 0.03389075 0.0006082796
## 124   124 0.03273215 0.18802725 0.02537419 0.0010355374 0.03390472 0.0006058616
## 125   125 0.03273460 0.18790452 0.02537577 0.0010340742 0.03375031 0.0006041654
## 126   126 0.03273081 0.18807814 0.02537257 0.0010313721 0.03379566 0.0006016746
## 127   127 0.03273348 0.18795871 0.02537387 0.0010304346 0.03391211 0.0006032236
## 128   128 0.03273482 0.18789408 0.02537281 0.0010306645 0.03393882 0.0006056804
## 129   129 0.03273502 0.18789667 0.02537506 0.0010317609 0.03392485 0.0006048815
## 130   130 0.03273430 0.18795268 0.02537367 0.0010344937 0.03411680 0.0006099635
## 131   131 0.03273050 0.18812428 0.02536982 0.0010365357 0.03428766 0.0006125780
## 132   132 0.03272728 0.18826976 0.02536604 0.0010376921 0.03440363 0.0006137921
## 133   133 0.03272652 0.18829927 0.02536554 0.0010345628 0.03436332 0.0006095151
## 134   134 0.03272717 0.18824400 0.02536648 0.0010316031 0.03413291 0.0006063551
## 135   135 0.03272861 0.18818103 0.02536766 0.0010319885 0.03406399 0.0006070329
## 136   136 0.03272840 0.18817615 0.02536682 0.0010306691 0.03391950 0.0006046425
## 137   137 0.03272914 0.18815963 0.02536852 0.0010303920 0.03395362 0.0006029156
## 138   138 0.03272688 0.18825322 0.02536591 0.0010303373 0.03390046 0.0006014799
## 139   139 0.03272841 0.18818724 0.02536586 0.0010287661 0.03384073 0.0005987557
## 140   140 0.03272766 0.18822236 0.02536378 0.0010283025 0.03386597 0.0005988105
## 141   141 0.03272988 0.18812177 0.02536553 0.0010277518 0.03388808 0.0005997787
## 142   142 0.03272945 0.18813972 0.02536681 0.0010285129 0.03389507 0.0005996815
## 143   143 0.03273172 0.18803401 0.02536858 0.0010271318 0.03379862 0.0005983827
## 144   144 0.03273118 0.18805575 0.02536741 0.0010272286 0.03379135 0.0005984701
## 145   145 0.03273142 0.18803956 0.02536675 0.0010269212 0.03380784 0.0005989952
## 146   146 0.03273038 0.18808519 0.02536570 0.0010274374 0.03380405 0.0006001165
## 147   147 0.03273117 0.18805072 0.02536742 0.0010282407 0.03378384 0.0006002984
## 148   148 0.03273148 0.18803306 0.02536851 0.0010276387 0.03379042 0.0006005813
## 149   149 0.03273186 0.18801995 0.02536907 0.0010268925 0.03377052 0.0006005685
## 150   150 0.03273183 0.18801984 0.02536907 0.0010266433 0.03376543 0.0005999739
## 151   151 0.03273162 0.18802704 0.02536856 0.0010262367 0.03374419 0.0006003124
## 152   152 0.03273137 0.18803677 0.02536861 0.0010261549 0.03374407 0.0005998615
## 153   153 0.03273170 0.18802270 0.02536903 0.0010260366 0.03374201 0.0006000006
## 154   154 0.03273175 0.18801994 0.02536914 0.0010263029 0.03375111 0.0006003751
## 155   155 0.03273174 0.18801812 0.02536919 0.0010261404 0.03374596 0.0006003774
## 156   156 0.03273170 0.18801970 0.02536912 0.0010260694 0.03374294 0.0006003263
## [1] "Best Model"
##    nvmax
## 60    60

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  2.096734e+00  2.095889e+00  2.097578e+00
## PC1          2.864714e-04  2.298585e-04  3.430843e-04
## PC2         -3.807254e-04 -4.475032e-04 -3.139476e-04
## PC3         -8.108879e-05 -1.610158e-04 -1.161790e-06
## PC4         -8.943715e-05 -1.717642e-04 -7.110109e-06
## PC5         -2.377410e-04 -3.203356e-04 -1.551464e-04
## PC6          1.066122e-04  2.221774e-05  1.910067e-04
## PC7         -1.681047e-04 -2.536784e-04 -8.253090e-05
## PC8         -1.304845e-04 -2.183161e-04 -4.265283e-05
## PC9          1.005409e-04  9.396046e-06  1.916858e-04
## PC11         4.782470e-04  3.808854e-04  5.756087e-04
## PC12        -3.662753e-04 -4.641971e-04 -2.683536e-04
## PC13         3.070715e-04  2.060052e-04  4.081378e-04
## PC14         5.442716e-04  4.419095e-04  6.466337e-04
## PC15        -1.126470e-04 -2.179876e-04 -7.306318e-06
## PC16         2.139724e-04  1.042408e-04  3.237040e-04
## PC20         4.000364e-04  2.677662e-04  5.323067e-04
## PC21        -3.549027e-04 -4.930481e-04 -2.167573e-04
## PC22         1.860156e-03  1.714040e-03  2.006272e-03
## PC23        -5.033151e-04 -9.217876e-04 -8.484262e-05
## PC25         4.304268e-04 -3.069138e-05  8.915450e-04
## PC37         6.740412e-04  5.728323e-05  1.290799e-03
## PC39        -7.536803e-04 -1.373151e-03 -1.342099e-04
## PC47        -6.216417e-04 -1.265883e-03  2.260009e-05
## PC50         6.637683e-04  6.747998e-06  1.320789e-03
## PC52        -1.184340e-03 -1.851822e-03 -5.168588e-04
## PC57         5.712924e-04 -1.157284e-04  1.258313e-03
## PC58         1.048528e-03  3.606851e-04  1.736372e-03
## PC61         8.429992e-04  1.466646e-04  1.539334e-03
## PC63         6.993862e-04 -1.444877e-07  1.398917e-03
## PC67        -6.397041e-04 -1.352111e-03  7.270237e-05
## PC69         1.044065e-03  3.339378e-04  1.754192e-03
## PC72         9.661565e-04  2.474483e-04  1.684865e-03
## PC77         1.105688e-03  3.790291e-04  1.832347e-03
## PC79        -1.009839e-03 -1.736829e-03 -2.828478e-04
## PC85        -5.895932e-04 -1.324399e-03  1.452122e-04
## PC87        -1.047616e-03 -1.782099e-03 -3.131339e-04
## PC90        -6.935265e-04 -1.432596e-03  4.554267e-05
## PC93         6.296589e-04 -1.177987e-04  1.377116e-03
## PC98         6.428428e-04 -1.070484e-04  1.392734e-03
## PC99        -1.363831e-03 -2.115953e-03 -6.117087e-04
## PC107       -1.270600e-03 -2.028160e-03 -5.130396e-04
## PC111       -8.857512e-04 -1.648669e-03 -1.228339e-04
## PC112       -1.168778e-03 -1.932519e-03 -4.050357e-04
## PC115       -8.263315e-04 -1.594540e-03 -5.812314e-05
## PC116        1.385187e-03  6.246659e-04  2.145707e-03
## PC117        8.163043e-04  4.737984e-05  1.585229e-03
## PC118        9.866072e-04  2.207404e-04  1.752474e-03
## PC122       -1.390951e-03 -2.157106e-03 -6.247957e-04
## PC126        9.561386e-04  1.766049e-04  1.735672e-03
## PC127       -1.070588e-03 -1.850154e-03 -2.910221e-04
## PC133       -1.214145e-03 -1.995517e-03 -4.327725e-04
## PC134        6.879742e-04 -9.600879e-05  1.471957e-03
## PC137       -8.324059e-04 -1.625185e-03 -3.962735e-05
## PC139       -7.629960e-04 -1.558015e-03  3.202295e-05
## PC143        1.136332e-03  3.444744e-04  1.928190e-03
## PC144        6.994141e-04 -9.858229e-05  1.497410e-03
## PC146        2.334270e-03  1.535459e-03  3.133081e-03
## PC152       -1.296909e-03 -2.097718e-03 -4.961007e-04
## PC153       -6.618241e-04 -1.462808e-03  1.391600e-04
## PC154        7.113556e-04 -9.418126e-05  1.516892e-03

Test

if (algo.backward.caret == TRUE){
  test.model(model.backward, data.test
             ,method = 'leapBackward',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.034   2.085   2.097   2.097   2.110   2.148 
## [1] "leapBackward  Test MSE: 0.00105127042865879"

Stepwise Selection with CV

Train

if (algo.stepwise.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "leapSeq"
                                   ,feature.names = feature.names)
  model.stepwise = returned$model
  id = returned$id
}
## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 21 on full training set
## [1] "All models results"
##    nvmax       RMSE   Rsquared        MAE       RMSESD  RsquaredSD        MAESD
## 1      1 0.03596827 0.01527512 0.02784873 0.0008825615 0.006983967 0.0004973760
## 2      2 0.03575392 0.02743041 0.02768393 0.0008225004 0.012005307 0.0003996057
## 3      3 0.03559100 0.03645637 0.02760432 0.0007565589 0.015046580 0.0003491449
## 4      4 0.03521623 0.05820690 0.02725501 0.0008120500 0.026133041 0.0003377191
## 5      5 0.03504569 0.06691536 0.02714897 0.0008683118 0.026560799 0.0003921311
## 6      6 0.03502016 0.06788875 0.02713994 0.0008683515 0.024237027 0.0003823853
## 7      7 0.03494874 0.07154979 0.02709469 0.0008759187 0.023547787 0.0003737225
## 8      8 0.03485154 0.07691164 0.02701881 0.0008845557 0.025484354 0.0004071258
## 9      9 0.03474539 0.08175798 0.02694702 0.0008798285 0.022240966 0.0004188303
## 10    10 0.03473831 0.08205722 0.02691589 0.0008735658 0.021275990 0.0004221505
## 11    11 0.03467281 0.08545058 0.02687814 0.0008590112 0.021741633 0.0004367692
## 12    12 0.03480051 0.07838011 0.02698333 0.0007736157 0.022396497 0.0003830715
## 13    13 0.03479003 0.07967316 0.02698113 0.0010144558 0.025800714 0.0005416168
## 14    14 0.03470861 0.08427002 0.02685605 0.0008561926 0.022475789 0.0003582521
## 15    15 0.03472035 0.08370827 0.02688634 0.0008174997 0.022799803 0.0003234209
## 16    16 0.03473846 0.08240525 0.02693034 0.0008690334 0.023843643 0.0004312217
## 17    17 0.03472856 0.08272831 0.02693944 0.0008762592 0.022490018 0.0004255057
## 18    18 0.03470802 0.08365234 0.02687973 0.0008204037 0.023552122 0.0004172611
## 19    19 0.03471436 0.08330441 0.02689963 0.0007927295 0.019590131 0.0003959860
## 20    20 0.03464868 0.08701104 0.02683826 0.0008400387 0.023194229 0.0003896163
## 21    21 0.03463457 0.08762411 0.02685159 0.0008402589 0.022592445 0.0004321247
## [1] "Best Model"
##    nvmax
## 21    21

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  2.096700e+00  2.095793e+00  2.097608e+00
## PC1          2.873471e-04  2.265370e-04  3.481572e-04
## PC2         -3.749128e-04 -4.466321e-04 -3.031935e-04
## PC3         -8.988881e-05 -1.757190e-04 -4.058607e-06
## PC4         -9.456845e-05 -1.829814e-04 -6.155477e-06
## PC5         -2.370038e-04 -3.257068e-04 -1.483009e-04
## PC6          9.804339e-05  7.424745e-06  1.886620e-04
## PC7         -1.746993e-04 -2.666232e-04 -8.277543e-05
## PC8         -1.275751e-04 -2.219056e-04 -3.324452e-05
## PC9          8.338440e-05 -1.453646e-05  1.813053e-04
## PC10        -2.575589e-05 -1.266890e-04  7.517724e-05
## PC11         4.798348e-04  3.752480e-04  5.844216e-04
## PC12        -3.711748e-04 -4.763764e-04 -2.659731e-04
## PC13         2.934464e-04  1.849049e-04  4.019878e-04
## PC14         5.422023e-04  4.323119e-04  6.520927e-04
## PC15        -1.240986e-04 -2.372591e-04 -1.093813e-05
## PC16         2.136898e-04  9.582792e-05  3.315516e-04
## PC17        -9.631725e-05 -2.188520e-04  2.621754e-05
## PC18        -8.540462e-05 -2.123825e-04  4.157327e-05
## PC19        -8.935724e-05 -2.220067e-04  4.329220e-05
## PC20         4.024553e-04  2.603635e-04  5.445472e-04
## PC21        -3.428740e-04 -4.912106e-04 -1.945375e-04

Test

if (algo.stepwise.caret == TRUE){
  test.model(model.stepwise, data.test
             ,method = 'leapSeq',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
  
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.066   2.089   2.097   2.097   2.105   2.137 
## [1] "leapSeq  Test MSE: 0.00114591004507159"

LASSO with CV

Train

if (algo.LASSO.caret == TRUE){
  set.seed(1)
  tune.grid= expand.grid(alpha = 1,lambda = 10^seq(from=-4,to=-2,length=100))
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "glmnet"
                                   ,subopt = 'LASSO'
                                   ,tune.grid = tune.grid
                                   ,feature.names = feature.names)
  model.LASSO.caret = returned$model
}
## Aggregating results
## Selecting tuning parameters
## Fitting alpha = 1, lambda = 0.000278 on full training set
## glmnet 
## 
## 5584 samples
##  156 predictor
## 
## No pre-processing
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 5026, 5026, 5026, 5025, 5025, 5026, ... 
## Resampling results across tuning parameters:
## 
##   lambda        RMSE        Rsquared    MAE       
##   0.0001000000  0.03264617  0.19050061  0.02531753
##   0.0001047616  0.03264333  0.19058621  0.02531631
##   0.0001097499  0.03264049  0.19067098  0.02531518
##   0.0001149757  0.03263765  0.19075605  0.02531403
##   0.0001204504  0.03263479  0.19084240  0.02531286
##   0.0001261857  0.03263191  0.19092996  0.02531162
##   0.0001321941  0.03262895  0.19102159  0.02531030
##   0.0001384886  0.03262595  0.19111612  0.02530898
##   0.0001450829  0.03262298  0.19120996  0.02530776
##   0.0001519911  0.03261995  0.19130827  0.02530658
##   0.0001592283  0.03261689  0.19140985  0.02530542
##   0.0001668101  0.03261392  0.19150902  0.02530440
##   0.0001747528  0.03261100  0.19160861  0.02530339
##   0.0001830738  0.03260814  0.19170855  0.02530243
##   0.0001917910  0.03260543  0.19180414  0.02530163
##   0.0002009233  0.03260285  0.19189897  0.02530097
##   0.0002104904  0.03260041  0.19199124  0.02530034
##   0.0002205131  0.03259822  0.19207768  0.02529981
##   0.0002310130  0.03259632  0.19215599  0.02529963
##   0.0002420128  0.03259478  0.19222372  0.02529978
##   0.0002535364  0.03259377  0.19227299  0.02530053
##   0.0002656088  0.03259310  0.19231464  0.02530148
##   0.0002782559  0.03259290  0.19234147  0.02530265
##   0.0002915053  0.03259345  0.19233991  0.02530446
##   0.0003053856  0.03259459  0.19231852  0.02530710
##   0.0003199267  0.03259628  0.19228169  0.02531004
##   0.0003351603  0.03259877  0.19221696  0.02531358
##   0.0003511192  0.03260168  0.19214680  0.02531752
##   0.0003678380  0.03260504  0.19206792  0.02532173
##   0.0003853529  0.03260895  0.19197809  0.02532603
##   0.0004037017  0.03261348  0.19187160  0.02533083
##   0.0004229243  0.03261894  0.19173342  0.02533606
##   0.0004430621  0.03262546  0.19155742  0.02534127
##   0.0004641589  0.03263330  0.19132763  0.02534727
##   0.0004862602  0.03264231  0.19105439  0.02535415
##   0.0005094138  0.03265283  0.19071994  0.02536249
##   0.0005336699  0.03266533  0.19030022  0.02537229
##   0.0005590810  0.03267939  0.18982037  0.02538297
##   0.0005857021  0.03269533  0.18925866  0.02539512
##   0.0006135907  0.03271238  0.18866657  0.02540827
##   0.0006428073  0.03273083  0.18801622  0.02542194
##   0.0006734151  0.03275060  0.18731692  0.02543636
##   0.0007054802  0.03277277  0.18650719  0.02545315
##   0.0007390722  0.03279569  0.18569250  0.02546972
##   0.0007742637  0.03281986  0.18483097  0.02548716
##   0.0008111308  0.03284570  0.18390046  0.02550662
##   0.0008497534  0.03287301  0.18290555  0.02552745
##   0.0008902151  0.03290287  0.18179441  0.02555105
##   0.0009326033  0.03293470  0.18058275  0.02557667
##   0.0009770100  0.03296788  0.17932061  0.02560348
##   0.0010235310  0.03300172  0.17802519  0.02563139
##   0.0010722672  0.03303755  0.17662886  0.02566159
##   0.0011233240  0.03307408  0.17519814  0.02569179
##   0.0011768120  0.03311236  0.17368135  0.02572281
##   0.0012328467  0.03315216  0.17207884  0.02575507
##   0.0012915497  0.03319215  0.17049148  0.02578759
##   0.0013530478  0.03323174  0.16892805  0.02581893
##   0.0014174742  0.03327172  0.16739570  0.02585047
##   0.0014849683  0.03331209  0.16585601  0.02588300
##   0.0015556761  0.03335248  0.16437538  0.02591443
##   0.0016297508  0.03339369  0.16286823  0.02594559
##   0.0017073526  0.03343780  0.16121539  0.02597811
##   0.0017886495  0.03348485  0.15939740  0.02601254
##   0.0018738174  0.03353565  0.15736699  0.02604923
##   0.0019630407  0.03359084  0.15506051  0.02608890
##   0.0020565123  0.03365030  0.15247751  0.02613131
##   0.0021544347  0.03371297  0.14965928  0.02617645
##   0.0022570197  0.03377439  0.14699863  0.02622093
##   0.0023644894  0.03383637  0.14427806  0.02626614
##   0.0024770764  0.03389602  0.14178557  0.02631099
##   0.0025950242  0.03395421  0.13939413  0.02635518
##   0.0027185882  0.03401455  0.13689149  0.02640045
##   0.0028480359  0.03407805  0.13417391  0.02644955
##   0.0029836472  0.03414600  0.13112237  0.02650256
##   0.0031257158  0.03421867  0.12767571  0.02655865
##   0.0032745492  0.03429249  0.12416060  0.02661536
##   0.0034304693  0.03436962  0.12032634  0.02667377
##   0.0035938137  0.03445246  0.11595031  0.02673546
##   0.0037649358  0.03454249  0.11084384  0.02680221
##   0.0039442061  0.03462973  0.10589733  0.02686471
##   0.0041320124  0.03471833  0.10067431  0.02692702
##   0.0043287613  0.03479782  0.09625030  0.02698389
##   0.0045348785  0.03486557  0.09266316  0.02703174
##   0.0047508102  0.03491900  0.09058711  0.02706985
##   0.0049770236  0.03496120  0.08955919  0.02710002
##   0.0052140083  0.03500055  0.08910629  0.02712801
##   0.0054622772  0.03503852  0.08910629  0.02715467
##   0.0057223677  0.03508015  0.08910629  0.02718372
##   0.0059948425  0.03512576  0.08910629  0.02721502
##   0.0062802914  0.03517575  0.08910629  0.02724997
##   0.0065793322  0.03523051  0.08910629  0.02728868
##   0.0068926121  0.03529051  0.08910629  0.02733148
##   0.0072208090  0.03535624  0.08910629  0.02737821
##   0.0075646333  0.03542822  0.08910629  0.02742987
##   0.0079248290  0.03550704  0.08910629  0.02748614
##   0.0083021757  0.03559334  0.08910629  0.02754791
##   0.0086974900  0.03568779  0.08910629  0.02761688
##   0.0091116276  0.03579116  0.08910629  0.02769299
##   0.0095454846  0.03590425  0.08910629  0.02777702
##   0.0100000000  0.03602794  0.08910629  0.02786833
## 
## Tuning parameter 'alpha' was held constant at a value of 1
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were alpha = 1 and lambda = 0.0002782559.

##    alpha       lambda
## 23     1 0.0002782559
##     alpha       lambda       RMSE   Rsquared        MAE       RMSESD RsquaredSD        MAESD
## 1       1 0.0001000000 0.03264617 0.19050061 0.02531753 0.0010391610 0.03483077 0.0005906667
## 2       1 0.0001047616 0.03264333 0.19058621 0.02531631 0.0010395479 0.03487220 0.0005899917
## 3       1 0.0001097499 0.03264049 0.19067098 0.02531518 0.0010399594 0.03491636 0.0005893319
## 4       1 0.0001149757 0.03263765 0.19075605 0.02531403 0.0010403800 0.03496237 0.0005886005
## 5       1 0.0001204504 0.03263479 0.19084240 0.02531286 0.0010407647 0.03500907 0.0005877888
## 6       1 0.0001261857 0.03263191 0.19092996 0.02531162 0.0010411697 0.03505669 0.0005870308
## 7       1 0.0001321941 0.03262895 0.19102159 0.02531030 0.0010415061 0.03510122 0.0005862387
## 8       1 0.0001384886 0.03262595 0.19111612 0.02530898 0.0010419170 0.03515079 0.0005855428
## 9       1 0.0001450829 0.03262298 0.19120996 0.02530776 0.0010424546 0.03520452 0.0005849023
## 10      1 0.0001519911 0.03261995 0.19130827 0.02530658 0.0010430320 0.03526267 0.0005842578
## 11      1 0.0001592283 0.03261689 0.19140985 0.02530542 0.0010436396 0.03532259 0.0005834866
## 12      1 0.0001668101 0.03261392 0.19150902 0.02530440 0.0010442398 0.03538114 0.0005826240
## 13      1 0.0001747528 0.03261100 0.19160861 0.02530339 0.0010448427 0.03543877 0.0005817181
## 14      1 0.0001830738 0.03260814 0.19170855 0.02530243 0.0010454926 0.03550444 0.0005806012
## 15      1 0.0001917910 0.03260543 0.19180414 0.02530163 0.0010461488 0.03557436 0.0005795718
## 16      1 0.0002009233 0.03260285 0.19189897 0.02530097 0.0010467939 0.03564716 0.0005785177
## 17      1 0.0002104904 0.03260041 0.19199124 0.02530034 0.0010475499 0.03572468 0.0005775232
## 18      1 0.0002205131 0.03259822 0.19207768 0.02529981 0.0010483553 0.03580390 0.0005763826
## 19      1 0.0002310130 0.03259632 0.19215599 0.02529963 0.0010492431 0.03588722 0.0005752031
## 20      1 0.0002420128 0.03259478 0.19222372 0.02529978 0.0010499656 0.03597489 0.0005734529
## 21      1 0.0002535364 0.03259377 0.19227299 0.02530053 0.0010510432 0.03607682 0.0005718132
## 22      1 0.0002656088 0.03259310 0.19231464 0.02530148 0.0010521113 0.03619361 0.0005700688
## 23      1 0.0002782559 0.03259290 0.19234147 0.02530265 0.0010530598 0.03631507 0.0005683893
## 24      1 0.0002915053 0.03259345 0.19233991 0.02530446 0.0010540278 0.03643232 0.0005666868
## 25      1 0.0003053856 0.03259459 0.19231852 0.02530710 0.0010547601 0.03654456 0.0005644804
## 26      1 0.0003199267 0.03259628 0.19228169 0.02531004 0.0010557592 0.03666872 0.0005625164
## 27      1 0.0003351603 0.03259877 0.19221696 0.02531358 0.0010572123 0.03680619 0.0005610059
## 28      1 0.0003511192 0.03260168 0.19214680 0.02531752 0.0010592133 0.03696316 0.0005601259
## 29      1 0.0003678380 0.03260504 0.19206792 0.02532173 0.0010608028 0.03711231 0.0005591131
## 30      1 0.0003853529 0.03260895 0.19197809 0.02532603 0.0010627777 0.03728188 0.0005584367
## 31      1 0.0004037017 0.03261348 0.19187160 0.02533083 0.0010644463 0.03743832 0.0005578645
## 32      1 0.0004229243 0.03261894 0.19173342 0.02533606 0.0010663633 0.03760743 0.0005570984
## 33      1 0.0004430621 0.03262546 0.19155742 0.02534127 0.0010682242 0.03777523 0.0005558241
## 34      1 0.0004641589 0.03263330 0.19132763 0.02534727 0.0010703049 0.03794906 0.0005545771
## 35      1 0.0004862602 0.03264231 0.19105439 0.02535415 0.0010722409 0.03810568 0.0005532214
## 36      1 0.0005094138 0.03265283 0.19071994 0.02536249 0.0010745165 0.03829049 0.0005525844
## 37      1 0.0005336699 0.03266533 0.19030022 0.02537229 0.0010769427 0.03848692 0.0005528051
## 38      1 0.0005590810 0.03267939 0.18982037 0.02538297 0.0010801213 0.03871799 0.0005539442
## 39      1 0.0005857021 0.03269533 0.18925866 0.02539512 0.0010828536 0.03894120 0.0005546813
## 40      1 0.0006135907 0.03271238 0.18866657 0.02540827 0.0010860756 0.03925308 0.0005557353
## 41      1 0.0006428073 0.03273083 0.18801622 0.02542194 0.0010882388 0.03953049 0.0005563015
## 42      1 0.0006734151 0.03275060 0.18731692 0.02543636 0.0010915404 0.03985858 0.0005584790
## 43      1 0.0007054802 0.03277277 0.18650719 0.02545315 0.0010953362 0.04020707 0.0005623500
## 44      1 0.0007390722 0.03279569 0.18569250 0.02546972 0.0010996221 0.04064877 0.0005683640
## 45      1 0.0007742637 0.03281986 0.18483097 0.02548716 0.0011043472 0.04106177 0.0005749255
## 46      1 0.0008111308 0.03284570 0.18390046 0.02550662 0.0011100872 0.04152137 0.0005820458
## 47      1 0.0008497534 0.03287301 0.18290555 0.02552745 0.0011159568 0.04191968 0.0005889936
## 48      1 0.0008902151 0.03290287 0.18179441 0.02555105 0.0011223390 0.04235460 0.0005955250
## 49      1 0.0009326033 0.03293470 0.18058275 0.02557667 0.0011270351 0.04269561 0.0006013460
## 50      1 0.0009770100 0.03296788 0.17932061 0.02560348 0.0011310184 0.04310947 0.0006071993
## 51      1 0.0010235310 0.03300172 0.17802519 0.02563139 0.0011318220 0.04336018 0.0006109515
## 52      1 0.0010722672 0.03303755 0.17662886 0.02566159 0.0011325507 0.04354883 0.0006142600
## 53      1 0.0011233240 0.03307408 0.17519814 0.02569179 0.0011315167 0.04355640 0.0006141945
## 54      1 0.0011768120 0.03311236 0.17368135 0.02572281 0.0011302316 0.04357305 0.0006135238
## 55      1 0.0012328467 0.03315216 0.17207884 0.02575507 0.0011264677 0.04343639 0.0006096628
## 56      1 0.0012915497 0.03319215 0.17049148 0.02578759 0.0011239462 0.04338016 0.0006064364
## 57      1 0.0013530478 0.03323174 0.16892805 0.02581893 0.0011182308 0.04306682 0.0006003362
## 58      1 0.0014174742 0.03327172 0.16739570 0.02585047 0.0011142836 0.04289795 0.0005965800
## 59      1 0.0014849683 0.03331209 0.16585601 0.02588300 0.0011091016 0.04253441 0.0005931215
## 60      1 0.0015556761 0.03335248 0.16437538 0.02591443 0.0011043064 0.04237517 0.0005899353
## 61      1 0.0016297508 0.03339369 0.16286823 0.02594559 0.0010971908 0.04203414 0.0005844191
## 62      1 0.0017073526 0.03343780 0.16121539 0.02597811 0.0010911044 0.04169251 0.0005793136
## 63      1 0.0017886495 0.03348485 0.15939740 0.02601254 0.0010844155 0.04122444 0.0005725948
## 64      1 0.0018738174 0.03353565 0.15736699 0.02604923 0.0010786179 0.04072598 0.0005663857
## 65      1 0.0019630407 0.03359084 0.15506051 0.02608890 0.0010731706 0.04014885 0.0005601347
## 66      1 0.0020565123 0.03365030 0.15247751 0.02613131 0.0010680984 0.03951570 0.0005546348
## 67      1 0.0021544347 0.03371297 0.14965928 0.02617645 0.0010613887 0.03861427 0.0005486449
## 68      1 0.0022570197 0.03377439 0.14699863 0.02622093 0.0010598743 0.03815163 0.0005468074
## 69      1 0.0023644894 0.03383637 0.14427806 0.02626614 0.0010577537 0.03737807 0.0005460818
## 70      1 0.0024770764 0.03389602 0.14178557 0.02631099 0.0010576567 0.03696160 0.0005493582
## 71      1 0.0025950242 0.03395421 0.13939413 0.02635518 0.0010543863 0.03615023 0.0005509180
## 72      1 0.0027185882 0.03401455 0.13689149 0.02640045 0.0010513409 0.03544244 0.0005523349
## 73      1 0.0028480359 0.03407805 0.13417391 0.02644955 0.0010480716 0.03455029 0.0005528786
## 74      1 0.0029836472 0.03414600 0.13112237 0.02650256 0.0010459556 0.03355527 0.0005545817
## 75      1 0.0031257158 0.03421867 0.12767571 0.02655865 0.0010451349 0.03236190 0.0005588777
## 76      1 0.0032745492 0.03429249 0.12416060 0.02661536 0.0010463143 0.03148805 0.0005638633
## 77      1 0.0034304693 0.03436962 0.12032634 0.02667377 0.0010475164 0.03036571 0.0005674295
## 78      1 0.0035938137 0.03445246 0.11595031 0.02673546 0.0010474071 0.02921814 0.0005706782
## 79      1 0.0037649358 0.03454249 0.11084384 0.02680221 0.0010470303 0.02788112 0.0005741424
## 80      1 0.0039442061 0.03462973 0.10589733 0.02686471 0.0010454088 0.02695443 0.0005778314
## 81      1 0.0041320124 0.03471833 0.10067431 0.02692702 0.0010454736 0.02581079 0.0005835776
## 82      1 0.0043287613 0.03479782 0.09625030 0.02698389 0.0010484332 0.02569577 0.0005897269
## 83      1 0.0045348785 0.03486557 0.09266316 0.02703174 0.0010476059 0.02443738 0.0005894391
## 84      1 0.0047508102 0.03491900 0.09058711 0.02706985 0.0010453591 0.02480725 0.0005862564
## 85      1 0.0049770236 0.03496120 0.08955919 0.02710002 0.0010357366 0.02437395 0.0005785413
## 86      1 0.0052140083 0.03500055 0.08910629 0.02712801 0.0010297358 0.02445943 0.0005733879
## 87      1 0.0054622772 0.03503852 0.08910629 0.02715467 0.0010256853 0.02445943 0.0005696828
## 88      1 0.0057223677 0.03508015 0.08910629 0.02718372 0.0010214948 0.02445943 0.0005657025
## 89      1 0.0059948425 0.03512576 0.08910629 0.02721502 0.0010171657 0.02445943 0.0005618714
## 90      1 0.0062802914 0.03517575 0.08910629 0.02724997 0.0010127006 0.02445943 0.0005580693
## 91      1 0.0065793322 0.03523051 0.08910629 0.02728868 0.0010081035 0.02445943 0.0005532795
## 92      1 0.0068926121 0.03529051 0.08910629 0.02733148 0.0010033804 0.02445943 0.0005486457
## 93      1 0.0072208090 0.03535624 0.08910629 0.02737821 0.0009985389 0.02445943 0.0005436583
## 94      1 0.0075646333 0.03542822 0.08910629 0.02742987 0.0009935895 0.02445943 0.0005383811
## 95      1 0.0079248290 0.03550704 0.08910629 0.02748614 0.0009885451 0.02445943 0.0005329255
## 96      1 0.0083021757 0.03559334 0.08910629 0.02754791 0.0009834218 0.02445943 0.0005277015
## 97      1 0.0086974900 0.03568779 0.08910629 0.02761688 0.0009782393 0.02445943 0.0005218109
## 98      1 0.0091116276 0.03579116 0.08910629 0.02769299 0.0009730217 0.02445943 0.0005159837
## 99      1 0.0095454846 0.03590425 0.08910629 0.02777702 0.0009677975 0.02445943 0.0005114156
## 100     1 0.0100000000 0.03602794 0.08910629 0.02786833 0.0009626004 0.02445943 0.0005075965

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients"
##                model.coef
## (Intercept)  2.096711e+00
## PC1          2.666222e-04
## PC2         -3.579258e-04
## PC3         -5.704718e-05
## PC4         -6.220090e-05
## PC5         -2.089476e-04
## PC6          7.693112e-05
## PC7         -1.405524e-04
## PC8         -1.023950e-04
## PC9          6.943889e-05
## PC11         4.414004e-04
## PC12        -3.323702e-04
## PC13         2.706144e-04
## PC14         5.090926e-04
## PC15        -7.901322e-05
## PC16         1.740467e-04
## PC17        -4.365366e-05
## PC18        -4.859943e-05
## PC19        -5.171185e-05
## PC20         3.554498e-04
## PC21        -3.096025e-04
## PC22         1.809404e-03
## PC23        -3.812967e-04
## PC25         2.685874e-04
## PC27        -2.008531e-06
## PC28        -4.254256e-07
## PC29         8.401885e-05
## PC31        -1.725935e-05
## PC32         1.400186e-06
## PC34         1.710036e-04
## PC37         4.292158e-04
## PC38         1.477531e-04
## PC39        -5.405397e-04
## PC41         1.674620e-04
## PC42         4.440227e-05
## PC45         1.902803e-04
## PC46         6.426631e-05
## PC47        -3.929390e-04
## PC48         1.513346e-04
## PC49         6.238624e-05
## PC50         4.447532e-04
## PC51        -1.513102e-04
## PC52        -9.707285e-04
## PC53        -8.930141e-05
## PC55         1.279354e-04
## PC56        -1.460454e-04
## PC57         3.356608e-04
## PC58         8.314150e-04
## PC59         5.921425e-05
## PC61         5.985997e-04
## PC62         2.067079e-04
## PC63         4.395940e-04
## PC64         2.827211e-05
## PC67        -4.091865e-04
## PC69         8.392032e-04
## PC70         1.382275e-04
## PC72         7.217962e-04
## PC74         1.202940e-04
## PC75         1.962659e-04
## PC76        -9.753797e-05
## PC77         8.836147e-04
## PC78        -1.593700e-05
## PC79        -7.468496e-04
## PC80        -1.628806e-04
## PC81         1.326114e-04
## PC83        -1.391984e-04
## PC84         9.073742e-07
## PC85        -3.458962e-04
## PC86        -1.487096e-04
## PC87        -7.891685e-04
## PC88         8.998984e-07
## PC89         1.811490e-04
## PC90        -4.775714e-04
## PC91         2.741873e-04
## PC93         3.617266e-04
## PC95         2.265874e-04
## PC96        -7.764702e-05
## PC98         3.973032e-04
## PC99        -1.144240e-03
## PC100       -2.654442e-04
## PC101        2.812385e-05
## PC102        6.682435e-05
## PC103       -2.763091e-04
## PC105       -1.945439e-04
## PC107       -1.047838e-03
## PC108        6.802262e-05
## PC109        1.827916e-04
## PC111       -6.207813e-04
## PC112       -9.136307e-04
## PC113        2.936782e-04
## PC115       -5.709988e-04
## PC116        1.129284e-03
## PC117        5.580494e-04
## PC118        7.323012e-04
## PC119       -8.790013e-05
## PC120        3.241205e-04
## PC122       -1.143715e-03
## PC126        7.141219e-04
## PC127       -8.246701e-04
## PC130        3.316504e-04
## PC133       -9.641013e-04
## PC134        4.276815e-04
## PC137       -6.004540e-04
## PC139       -5.074014e-04
## PC140        9.236710e-05
## PC143        8.755517e-04
## PC144        4.392563e-04
## PC146        2.058305e-03
## PC148       -2.186550e-04
## PC150       -3.160812e-04
## PC152       -1.022602e-03
## PC153       -3.643977e-04
## PC154        4.195155e-04
## PC155       -2.588174e-04
## PC156        6.028137e-05

Test

if (algo.LASSO.caret == TRUE){
  test.model(model.LASSO.caret, data.test
             ,method = 'glmnet',subopt = "LASSO"
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.038   2.086   2.098   2.097   2.109   2.145 
## [1] "glmnet LASSO Test MSE: 0.00104134602186552"

LARS with CV

Train

if (algo.LARS.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "lars"
                                   ,subopt = 'NULL'
                                   ,feature.names = feature.names)
  model.LARS.caret = returned$model
}
## Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, : There were missing values in resampled
## performance measures.
## Aggregating results
## Selecting tuning parameters
## Fitting fraction = 0.707 on full training set
## Least Angle Regression 
## 
## 5584 samples
##  156 predictor
## 
## Pre-processing: centered (156), scaled (156) 
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 5026, 5026, 5026, 5025, 5025, 5026, ... 
## Resampling results across tuning parameters:
## 
##   fraction    RMSE        Rsquared    MAE       
##   0.00000000  0.03622572         NaN  0.02801456
##   0.01010101  0.03587006  0.08910629  0.02775169
##   0.02020202  0.03555628  0.08910629  0.02752135
##   0.03030303  0.03528550  0.08910629  0.02732768
##   0.04040404  0.03505871  0.08910629  0.02716880
##   0.05050505  0.03488316  0.09168749  0.02704436
##   0.06060606  0.03473540  0.09964768  0.02693914
##   0.07070707  0.03459549  0.10785340  0.02684093
##   0.08080808  0.03445751  0.11587554  0.02673886
##   0.09090909  0.03432847  0.12257774  0.02664283
##   0.10101010  0.03421147  0.12814853  0.02655339
##   0.11111111  0.03410065  0.13328290  0.02646765
##   0.12121212  0.03399652  0.13771432  0.02638656
##   0.13131313  0.03390751  0.14128467  0.02631974
##   0.14141414  0.03382593  0.14467226  0.02625962
##   0.15151515  0.03374569  0.14822603  0.02620124
##   0.16161616  0.03366726  0.15176400  0.02614452
##   0.17171717  0.03359215  0.15503013  0.02609047
##   0.18181818  0.03352172  0.15794230  0.02604010
##   0.19191919  0.03345685  0.16050583  0.02599406
##   0.20202020  0.03340085  0.16259297  0.02595359
##   0.21212121  0.03335202  0.16434276  0.02591657
##   0.22222222  0.03330722  0.16600151  0.02588052
##   0.23232323  0.03326651  0.16755078  0.02584744
##   0.24242424  0.03322857  0.16904682  0.02581717
##   0.25252525  0.03319242  0.17047510  0.02578871
##   0.26262626  0.03315798  0.17182949  0.02576122
##   0.27272727  0.03312499  0.17313994  0.02573437
##   0.28282828  0.03309349  0.17439558  0.02570920
##   0.29292929  0.03306260  0.17563051  0.02568338
##   0.30303030  0.03303404  0.17674625  0.02565927
##   0.31313131  0.03300663  0.17781649  0.02563616
##   0.32323232  0.03298012  0.17884287  0.02561384
##   0.33333333  0.03295453  0.17982268  0.02559271
##   0.34343434  0.03292979  0.18077922  0.02557257
##   0.35353535  0.03290610  0.18168933  0.02555339
##   0.36363636  0.03288399  0.18252342  0.02553566
##   0.37373737  0.03286320  0.18330631  0.02551917
##   0.38383838  0.03284351  0.18403870  0.02550396
##   0.39393939  0.03282539  0.18469869  0.02549021
##   0.40404040  0.03280811  0.18532107  0.02547777
##   0.41414141  0.03279177  0.18590494  0.02546606
##   0.42424242  0.03277633  0.18644989  0.02545481
##   0.43434343  0.03276116  0.18700122  0.02544363
##   0.44444444  0.03274685  0.18751797  0.02543311
##   0.45454545  0.03273340  0.18799465  0.02542327
##   0.46464646  0.03272081  0.18843700  0.02541425
##   0.47474747  0.03270892  0.18884833  0.02540528
##   0.48484848  0.03269717  0.18926500  0.02539619
##   0.49494949  0.03268635  0.18964145  0.02538795
##   0.50505051  0.03267587  0.19001067  0.02537990
##   0.51515152  0.03266617  0.19034821  0.02537235
##   0.52525253  0.03265731  0.19064865  0.02536557
##   0.53535354  0.03264937  0.19090993  0.02535936
##   0.54545455  0.03264234  0.19112741  0.02535361
##   0.55555556  0.03263591  0.19131906  0.02534839
##   0.56565657  0.03262990  0.19149676  0.02534387
##   0.57575758  0.03262446  0.19165210  0.02533983
##   0.58585859  0.03261969  0.19177961  0.02533600
##   0.59595960  0.03261527  0.19189623  0.02533230
##   0.60606061  0.03261135  0.19199246  0.02532864
##   0.61616162  0.03260807  0.19206698  0.02532521
##   0.62626263  0.03260523  0.19212530  0.02532213
##   0.63636364  0.03260248  0.19218446  0.02531901
##   0.64646465  0.03260010  0.19223165  0.02531586
##   0.65656566  0.03259772  0.19228997  0.02531274
##   0.66666667  0.03259558  0.19234365  0.02530983
##   0.67676768  0.03259379  0.19238804  0.02530706
##   0.68686869  0.03259227  0.19242551  0.02530452
##   0.69696970  0.03259166  0.19242525  0.02530272
##   0.70707071  0.03259148  0.19241019  0.02530137
##   0.71717172  0.03259158  0.19238714  0.02530033
##   0.72727273  0.03259201  0.19235382  0.02529942
##   0.73737374  0.03259286  0.19230664  0.02529861
##   0.74747475  0.03259424  0.19224121  0.02529820
##   0.75757576  0.03259612  0.19216075  0.02529830
##   0.76767677  0.03259847  0.19206367  0.02529877
##   0.77777778  0.03260109  0.19196236  0.02529948
##   0.78787879  0.03260410  0.19185176  0.02530030
##   0.79797980  0.03260743  0.19173238  0.02530128
##   0.80808081  0.03261107  0.19160626  0.02530271
##   0.81818182  0.03261504  0.19147127  0.02530418
##   0.82828283  0.03261935  0.19132618  0.02530598
##   0.83838384  0.03262362  0.19118942  0.02530765
##   0.84848485  0.03262818  0.19104692  0.02530967
##   0.85858586  0.03263299  0.19089918  0.02531181
##   0.86868687  0.03263792  0.19074977  0.02531388
##   0.87878788  0.03264317  0.19059301  0.02531601
##   0.88888889  0.03264890  0.19042055  0.02531859
##   0.89898990  0.03265493  0.19024135  0.02532135
##   0.90909091  0.03266130  0.19005233  0.02532465
##   0.91919192  0.03266801  0.18985243  0.02532818
##   0.92929293  0.03267515  0.18963945  0.02533212
##   0.93939394  0.03268250  0.18942299  0.02533642
##   0.94949495  0.03268999  0.18920674  0.02534106
##   0.95959596  0.03269774  0.18898459  0.02534619
##   0.96969697  0.03270579  0.18875475  0.02535147
##   0.97979798  0.03271415  0.18851592  0.02535696
##   0.98989899  0.03272282  0.18826960  0.02536285
##   1.00000000  0.03273170  0.18801970  0.02536912
## 
## RMSE was used to select the optimal model using the smallest value.
## The final value used for the model was fraction = 0.7070707.

##     fraction
## 71 0.7070707
## Warning: Removed 1 rows containing missing values (geom_point).

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients"
##           PC1           PC2           PC3           PC4           PC5           PC6           PC7           PC8 
##  3.979204e-03 -4.529032e-03 -6.041100e-04 -6.397054e-04 -2.138280e-03  7.716498e-04 -1.388493e-03 -9.861185e-04 
##           PC9          PC11          PC12          PC13          PC14          PC15          PC16          PC17 
##  6.446563e-04  3.831253e-03 -2.867385e-03  2.263494e-03  4.203485e-03 -6.344864e-04  1.341472e-03 -3.243570e-04 
##          PC18          PC19          PC20          PC21          PC22          PC23          PC25          PC29 
## -3.483269e-04 -3.548302e-04  2.270444e-03 -1.895473e-03  1.045870e-02 -7.708148e-04  4.930938e-04  1.395846e-04 
##          PC31          PC34          PC37          PC38          PC39          PC41          PC42          PC45 
## -2.702427e-05  2.440591e-04  5.895480e-04  2.041306e-04 -7.384314e-04  2.258879e-04  6.065922e-05  2.527388e-04 
##          PC46          PC47          PC48          PC49          PC50          PC51          PC52          PC53 
##  8.482054e-05 -5.163517e-04  1.964055e-04  8.212524e-05  5.727637e-04 -1.928417e-04 -1.230096e-03 -1.124879e-04 
##          PC55          PC56          PC57          PC58          PC59          PC61          PC62          PC63 
##  1.616172e-04 -1.806301e-04  4.140550e-04  1.022022e-03  7.369336e-05  7.274535e-04  2.516309e-04  5.319542e-04 
##          PC64          PC67          PC69          PC70          PC72          PC74          PC75          PC76 
##  3.540804e-05 -4.864265e-04  9.993174e-04  1.628052e-04  8.495550e-04  1.416114e-04  2.311658e-04 -1.148550e-04 
##          PC77          PC78          PC79          PC80          PC81          PC83          PC85          PC86 
##  1.028472e-03 -1.983963e-05 -8.689957e-04 -1.900813e-04  1.547005e-04 -1.613707e-04 -3.989476e-04 -1.709126e-04 
##          PC87          PC89          PC90          PC91          PC93          PC95          PC96          PC98 
## -9.085835e-04  2.083146e-04 -5.471581e-04  3.104819e-04  4.099485e-04  2.586501e-04 -8.919372e-05  4.487015e-04 
##          PC99         PC100         PC101         PC102         PC103         PC105         PC107         PC108 
## -1.286102e-03 -2.982126e-04  3.264192e-05  7.582708e-05 -3.102867e-04 -2.161974e-04 -1.169304e-03  7.688916e-05 
##         PC109         PC111         PC112         PC113         PC115         PC116         PC117         PC118 
##  2.018792e-04 -6.883543e-04 -1.011828e-03  3.266372e-04 -6.289574e-04  1.255450e-03  6.142997e-04  8.087562e-04 
##         PC119         PC120         PC122         PC126         PC127         PC130         PC133         PC134 
## -9.771394e-05  3.556935e-04 -1.261866e-03  7.747775e-04 -8.947624e-04  3.592737e-04 -1.043070e-03  4.618132e-04 
##         PC137         PC139         PC140         PC143         PC144         PC146         PC148         PC150 
## -6.409746e-04 -5.402092e-04  9.978505e-05  9.349360e-04  4.660234e-04  2.177732e-03 -2.316716e-04 -3.348448e-04 
##         PC152         PC153         PC154         PC155         PC156 
## -1.079600e-03 -3.857370e-04  4.412399e-04 -2.723591e-04  6.441860e-05

Test

if (algo.LARS.caret == TRUE){
  test.model(model.LARS.caret, data.test
             ,method = 'lars',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.038   2.086   2.098   2.097   2.109   2.145 
## [1] "lars  Test MSE: 0.00104132775293204"

Session Info

sessionInfo()
## R version 3.5.1 (2018-07-02)
## Platform: x86_64-w64-mingw32/x64 (64-bit)
## Running under: Windows 10 x64 (build 17134)
## 
## Matrix products: default
## 
## locale:
## [1] LC_COLLATE=English_United States.1252  LC_CTYPE=English_United States.1252    LC_MONETARY=English_United States.1252
## [4] LC_NUMERIC=C                           LC_TIME=English_United States.1252    
## 
## attached base packages:
## [1] parallel  stats     graphics  grDevices utils     datasets  methods   base     
## 
## other attached packages:
##  [1] bindrcpp_0.2.2             knitr_1.20                 htmltools_0.3.6            reshape2_1.4.3            
##  [5] lars_1.2                   doParallel_1.0.14          iterators_1.0.10           caret_6.0-81              
##  [9] leaps_3.0                  ggforce_0.1.3              rlist_0.4.6.1              car_3.0-2                 
## [13] carData_3.0-2              bestNormalize_1.3.0        scales_1.0.0               onewaytests_2.0           
## [17] caTools_1.17.1.1           mosaic_1.5.0               mosaicData_0.17.0          ggformula_0.9.1           
## [21] ggstance_0.3.1             lattice_0.20-35            DT_0.5                     ggiraph_0.6.0             
## [25] investr_1.4.0              glmnet_2.0-16              foreach_1.4.4              Matrix_1.2-14             
## [29] MASS_7.3-50                PerformanceAnalytics_1.5.2 xts_0.11-2                 zoo_1.8-4                 
## [33] forcats_0.3.0              stringr_1.3.1              dplyr_0.7.8                purrr_0.2.5               
## [37] readr_1.3.1                tidyr_0.8.2                tibble_1.4.2               ggplot2_3.1.0             
## [41] tidyverse_1.2.1            usdm_1.1-18                raster_2.8-4               sp_1.3-1                  
## [45] pacman_0.5.0              
## 
## loaded via a namespace (and not attached):
##  [1] readxl_1.2.0       backports_1.1.3    plyr_1.8.4         lazyeval_0.2.1     splines_3.5.1      mycor_0.1.1       
##  [7] crosstalk_1.0.0    leaflet_2.0.2      digest_0.6.18      magrittr_1.5       mosaicCore_0.6.0   openxlsx_4.1.0    
## [13] recipes_0.1.4      modelr_0.1.2       gower_0.1.2        colorspace_1.3-2   rvest_0.3.2        ggrepel_0.8.0     
## [19] haven_2.0.0        crayon_1.3.4       jsonlite_1.5       bindr_0.1.1        survival_2.42-3    glue_1.3.0        
## [25] registry_0.5       gtable_0.2.0       ppcor_1.1          ipred_0.9-8        abind_1.4-5        rngtools_1.3.1    
## [31] bibtex_0.4.2       Rcpp_1.0.0         xtable_1.8-3       units_0.6-2        foreign_0.8-70     stats4_3.5.1      
## [37] lava_1.6.4         prodlim_2018.04.18 htmlwidgets_1.3    httr_1.4.0         RColorBrewer_1.1-2 pkgconfig_2.0.2   
## [43] farver_1.1.0       nnet_7.3-12        labeling_0.3       tidyselect_0.2.5   rlang_0.3.1        later_0.7.5       
## [49] munsell_0.5.0      cellranger_1.1.0   tools_3.5.1        cli_1.0.1          generics_0.0.2     moments_0.14      
## [55] sjlabelled_1.0.17  broom_0.5.1        evaluate_0.12      ggdendro_0.1-20    yaml_2.2.0         ModelMetrics_1.2.2
## [61] zip_2.0.1          nlme_3.1-137       doRNG_1.7.1        mime_0.6           xml2_1.2.0         compiler_3.5.1    
## [67] rstudioapi_0.8     curl_3.2           tweenr_1.0.1       stringi_1.2.4      gdtools_0.1.7      pillar_1.3.1      
## [73] data.table_1.11.8  bitops_1.0-6       insight_0.1.2      httpuv_1.4.5       R6_2.3.0           promises_1.0.1    
## [79] gridExtra_2.3      rio_0.5.16         codetools_0.2-15   assertthat_0.2.0   pkgmaker_0.27      withr_2.1.2       
## [85] nortest_1.0-4      mgcv_1.8-24        hms_0.4.2          quadprog_1.5-5     grid_3.5.1         rpart_4.1-13      
## [91] timeDate_3043.102  class_7.3-14       rmarkdown_1.11     shiny_1.2.0        lubridate_1.7.4